Merge pull request #9676 from Security-Onion-Solutions/fix/so-import-evtx_updates

Updates to so-import-evtx

salt/common/tools/sbin/so-elastic-fleet-integration-policy-load

@@ -31,6 +31,13 @@ echo "Setting up Zeek package policy..."
 curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "name": "zeek-logs", "description": "Zeek logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": ["/nsm/zeek/logs/current/*.log"], "data_stream.dataset": "zeek", "tags": [], "processors": "- dissect:\n    tokenizer: \"/nsm/zeek/logs/current/%{pipeline}.log\"\n    field: \"log.file.path\"\n    trim_chars: \".log\"\n    target_prefix: \"\"\n- script:\n      lang: javascript\n      source: >\n        function process(event) {\n          var pl = event.Get(\"pipeline\");\n          event.Put(\"@metadata.pipeline\", \"zeek.\" + pl);\n        }\n- add_fields:\n    target: event\n    fields:\n      category: network\n      module: zeek\n- add_tags:\n    tags: \"ics\"\n    when:\n      regexp:\n        pipeline: \"^bacnet*|^bsap*|^cip*|^cotp*|^dnp3*|^ecat*|^enip*|^modbus*|^opcua*|^profinet*|^s7comm*\"", "custom": "exclude_files: [\"broker|capture_loss|ecat_arp_info|loaded_scripts|packet_filter|stats|stderr|stdout.log$\"]\n" } } } } } }'
 echo
 
+
+# Import - EVTX
+echo
+echo "Setting up EVTX import package policy..."
+curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{ "package": { "name": "log", "version": "1.1.0" }, "name": "import-evtx-logs", "namespace": "so", "description": "Import Windows EVTX logs", "policy_id": "so-grid-nodes", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/import/*/evtx/data.json" ], "data_stream.dataset": "import", "custom": "pipeline: import.wel", "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- add_fields:\n    target: event\n    fields:\n      module: windows_eventlog\n      imported: true", "tags": [] } } } } } }'
+echo
+
 # Import - Suricata logs
 echo
 echo "Setting up Suricata import package policy..."
@@ -44,6 +51,7 @@ curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POS
 echo
 
 # Strelka logs
+echo
 echo "Setting up Strelka package policy..."
 curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "strelka-logs", "name": "strelka-logs", "description": "Strelka logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/strelka/log/strelka.log" ], "data_stream.dataset": "file", "tags": [],"processors": "- add_fields:\n    target: event\n    fields:\n      category: file\n      module: strelka", "custom": "pipeline: strelka.file" }}}}}}'
 echo

salt/common/tools/sbin/so-import-evtx

@@ -32,24 +32,13 @@ function evtx2es() {
   EVTX=$1
   HASH=$2
   
-  ES_PASS=$(lookup_pillar "auth:users:so_elastic_user:pass" "elasticsearch")
-  ES_USER=$(lookup_pillar "auth:users:so_elastic_user:user" "elasticsearch")
-
   docker run --rm \
-    -v "$EVTX:/tmp/$RUNID.evtx" \
-    --entrypoint evtx2es \
-    {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-pcaptools:{{ VERSION }} \
-    --host {{ MANAGERIP }} --scheme https \
-    --index so-beats-$INDEX_DATE --pipeline import.wel \
-    --login $ES_USER --pwd "$ES_PASS" \
-    "/tmp/$RUNID.evtx" >> $LOG_FILE 2>&1
-
-  docker run --rm \
-    -v "$EVTX:/tmp/import.evtx" \
+    -v "$EVTX:/tmp/data.evtx" \
+    -v "/nsm/import/$HASH/evtx/:/tmp/evtx/" \
     -v "/nsm/import/evtx-end_newest:/tmp/newest" \
     -v "/nsm/import/evtx-start_oldest:/tmp/oldest" \
-    --entrypoint '/evtx_calc_timestamps.sh' \
-    {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-pcaptools:{{ VERSION }}
+    --entrypoint "/evtx_calc_timestamps.sh" \
+    {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-pcaptools:{{ VERSION }} >> $LOG_FILE 2>&1
 }
 
 # if no parameters supplied, display usage

salt/elasticsearch/files/ingest/import.wel

@@ -1,10 +1,11 @@
 {
   "description" : "import.wel",
   "processors" : [
-    { "remove":         { "field": ["event.created","timestamp", "winlog.event_data.UtcTime", "event_record_id"],     "ignore_failure": true                                               } },
-    { "pipeline":      { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational'",   "name": "sysmon"  }  },
-    { "pipeline":      { "if": "ctx.winlog?.channel != 'Microsoft-Windows-Sysmon/Operational'",  "name":"win.eventlogs" }  },
-    { "dissect":       { "field": "log.file.name", "pattern" : "/tmp/%{import.id}.evtx" } },
-    { "pipeline":    { "name": "common" } }
+    { "set":           { "field": "event.ingested", "value": "{{ @timestamp }}"  } },
+    { "set" :          { "field" : "@timestamp", "value" : "{{ event.created }}" } },
+    { "remove":        { "field": [ "event_record_id", "event.created" , "timestamp" , "winlog.event_data.UtcTime" ], "ignore_failure": true } },
+    { "pipeline":      { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational'",   "name": "sysmon" } },
+    { "pipeline":      { "if": "ctx.winlog?.channel != 'Microsoft-Windows-Sysmon/Operational'",  "name":"win.eventlogs" } },
+    { "pipeline":      { "name": "common" } }
   ]
-}
+}