diff --git a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load index ad82c8892..e82011896 100755 --- a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load +++ b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load @@ -31,6 +31,13 @@ echo "Setting up Zeek package policy..." curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "name": "zeek-logs", "description": "Zeek logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": ["/nsm/zeek/logs/current/*.log"], "data_stream.dataset": "zeek", "tags": [], "processors": "- dissect:\n tokenizer: \"/nsm/zeek/logs/current/%{pipeline}.log\"\n field: \"log.file.path\"\n trim_chars: \".log\"\n target_prefix: \"\"\n- script:\n lang: javascript\n source: >\n function process(event) {\n var pl = event.Get(\"pipeline\");\n event.Put(\"@metadata.pipeline\", \"zeek.\" + pl);\n }\n- add_fields:\n target: event\n fields:\n category: network\n module: zeek\n- add_tags:\n tags: \"ics\"\n when:\n regexp:\n pipeline: \"^bacnet*|^bsap*|^cip*|^cotp*|^dnp3*|^ecat*|^enip*|^modbus*|^opcua*|^profinet*|^s7comm*\"", "custom": "exclude_files: [\"broker|capture_loss|ecat_arp_info|loaded_scripts|packet_filter|stats|stderr|stdout.log$\"]\n" } } } } } }' echo + +# Import - EVTX +echo +echo "Setting up EVTX import package policy..." +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{ "package": { "name": "log", "version": "1.1.0" }, "name": "import-evtx-logs", "namespace": "so", "description": "Import Windows EVTX logs", "policy_id": "so-grid-nodes", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/import/*/evtx/data.json" ], "data_stream.dataset": "import", "custom": "pipeline: import.wel", "processors": "- dissect:\n tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"\n- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n- add_fields:\n target: event\n fields:\n module: windows_eventlog\n imported: true", "tags": [] } } } } } }' +echo + # Import - Suricata logs echo echo "Setting up Suricata import package policy..." @@ -44,6 +51,7 @@ curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POS echo # Strelka logs +echo echo "Setting up Strelka package policy..." curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "strelka-logs", "name": "strelka-logs", "description": "Strelka logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/strelka/log/strelka.log" ], "data_stream.dataset": "file", "tags": [],"processors": "- add_fields:\n target: event\n fields:\n category: file\n module: strelka", "custom": "pipeline: strelka.file" }}}}}}' echo diff --git a/salt/common/tools/sbin/so-import-evtx b/salt/common/tools/sbin/so-import-evtx index 0b6bd8383..edb0a08a8 100755 --- a/salt/common/tools/sbin/so-import-evtx +++ b/salt/common/tools/sbin/so-import-evtx @@ -32,24 +32,13 @@ function evtx2es() { EVTX=$1 HASH=$2 - ES_PASS=$(lookup_pillar "auth:users:so_elastic_user:pass" "elasticsearch") - ES_USER=$(lookup_pillar "auth:users:so_elastic_user:user" "elasticsearch") - docker run --rm \ - -v "$EVTX:/tmp/$RUNID.evtx" \ - --entrypoint evtx2es \ - {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-pcaptools:{{ VERSION }} \ - --host {{ MANAGERIP }} --scheme https \ - --index so-beats-$INDEX_DATE --pipeline import.wel \ - --login $ES_USER --pwd "$ES_PASS" \ - "/tmp/$RUNID.evtx" >> $LOG_FILE 2>&1 - - docker run --rm \ - -v "$EVTX:/tmp/import.evtx" \ + -v "$EVTX:/tmp/data.evtx" \ + -v "/nsm/import/$HASH/evtx/:/tmp/evtx/" \ -v "/nsm/import/evtx-end_newest:/tmp/newest" \ -v "/nsm/import/evtx-start_oldest:/tmp/oldest" \ - --entrypoint '/evtx_calc_timestamps.sh' \ - {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-pcaptools:{{ VERSION }} + --entrypoint "/evtx_calc_timestamps.sh" \ + {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-pcaptools:{{ VERSION }} >> $LOG_FILE 2>&1 } # if no parameters supplied, display usage diff --git a/salt/elasticsearch/files/ingest/import.wel b/salt/elasticsearch/files/ingest/import.wel index e75098f8f..d03cd34e4 100644 --- a/salt/elasticsearch/files/ingest/import.wel +++ b/salt/elasticsearch/files/ingest/import.wel @@ -1,10 +1,11 @@ { "description" : "import.wel", "processors" : [ - { "remove": { "field": ["event.created","timestamp", "winlog.event_data.UtcTime", "event_record_id"], "ignore_failure": true } }, - { "pipeline": { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational'", "name": "sysmon" } }, - { "pipeline": { "if": "ctx.winlog?.channel != 'Microsoft-Windows-Sysmon/Operational'", "name":"win.eventlogs" } }, - { "dissect": { "field": "log.file.name", "pattern" : "/tmp/%{import.id}.evtx" } }, - { "pipeline": { "name": "common" } } + { "set": { "field": "event.ingested", "value": "{{ @timestamp }}" } }, + { "set" : { "field" : "@timestamp", "value" : "{{ event.created }}" } }, + { "remove": { "field": [ "event_record_id", "event.created" , "timestamp" , "winlog.event_data.UtcTime" ], "ignore_failure": true } }, + { "pipeline": { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational'", "name": "sysmon" } }, + { "pipeline": { "if": "ctx.winlog?.channel != 'Microsoft-Windows-Sysmon/Operational'", "name":"win.eventlogs" } }, + { "pipeline": { "name": "common" } } ] -} \ No newline at end of file +}