From cfc232eafa4d7c480f2373c0750bdc1eae75d845 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 24 Jan 2023 10:23:42 -0500 Subject: [PATCH 01/10] try paths filter on both even though docs only mention support for push --- .github/workflows/pythontest.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pythontest.yml b/.github/workflows/pythontest.yml index 241245044..f888b1a43 100644 --- a/.github/workflows/pythontest.yml +++ b/.github/workflows/pythontest.yml @@ -1,6 +1,10 @@ name: python-test -on: [push, pull_request] +on: + push: + paths: salt/sensoroni/files/analyzers + pull_request: + paths: salt/sensoroni/files/analyzers jobs: build: From ec2e923530419d43d59895d69b0ab412f7e9c78a Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 24 Jan 2023 10:28:39 -0500 Subject: [PATCH 02/10] Add proper spacing between headers and content --- salt/sensoroni/files/analyzers/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/sensoroni/files/analyzers/README.md b/salt/sensoroni/files/analyzers/README.md index e87a95638..8b1f44f29 100644 --- a/salt/sensoroni/files/analyzers/README.md +++ b/salt/sensoroni/files/analyzers/README.md @@ -3,6 +3,7 @@ Security Onion provides a means for performing data analysis on varying inputs. This data can be any data of interest sourced from event logs. Examples include hostnames, IP addresses, file hashes, URLs, etc. The analysis is conducted by one or more analyzers that understand that type of input. Analyzers come with the default installation of Security Onion. However, it is also possible to add additional analyzers to extend the analysis across additional areas or data types. ## Supported Observable Types + The built-in analyzers support the following observable types: | Name | Domain | Hash | IP | Mail | Other | URI | URL | User Agent | @@ -20,6 +21,7 @@ The built-in analyzers support the following observable types: | WhoisLookup |✓ |✗|✗|✗|✗|✓|✗|✗| ## Authentication + Many analyzers require authentication, via an API key or similar. The table below illustrates which analyzers require authentication. | Name | Authn Req'd| From 62b96c3698907dec2feba42cc56d4990cd0f3a50 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 24 Jan 2023 10:31:02 -0500 Subject: [PATCH 03/10] rework filter for action --- .github/workflows/pythontest.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pythontest.yml b/.github/workflows/pythontest.yml index f888b1a43..dae1c420b 100644 --- a/.github/workflows/pythontest.yml +++ b/.github/workflows/pythontest.yml @@ -2,9 +2,9 @@ name: python-test on: push: - paths: salt/sensoroni/files/analyzers + paths: + - "salt/sensoroni/files/analyzers/**" pull_request: - paths: salt/sensoroni/files/analyzers jobs: build: From 0dc5e7e714caf1ed3e4b242f9c79625b6e254cba Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 24 Jan 2023 10:38:59 -0500 Subject: [PATCH 04/10] try paths with wildcard --- .github/workflows/pythontest.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pythontest.yml b/.github/workflows/pythontest.yml index dae1c420b..3ce481ee0 100644 --- a/.github/workflows/pythontest.yml +++ b/.github/workflows/pythontest.yml @@ -5,6 +5,8 @@ on: paths: - "salt/sensoroni/files/analyzers/**" pull_request: + paths: + - "salt/sensoroni/files/analyzers/**" jobs: build: From 1e5377c78ab26e366cf39280ab0b91665d16fc25 Mon Sep 17 00:00:00 2001 From: Wes Date: Tue, 24 Jan 2023 16:56:20 +0000 Subject: [PATCH 05/10] Condense RITA integration policies, add ICS tags, and improve output readability --- ...astic-fleet-integration-policy-bulk-delete | 2 + .../so-elastic-fleet-integration-policy-load | 48 +++++++++++++------ 2 files changed, 35 insertions(+), 15 deletions(-) diff --git a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-bulk-delete b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-bulk-delete index 9c2061cd9..4596774b4 100644 --- a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-bulk-delete +++ b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-bulk-delete @@ -18,4 +18,6 @@ for i in $INTEGRATION_POLICY_IDS; do # Delete integration policies echo "Deleting integration policy: $i..." curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies/delete" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d"{\"packagePolicyIds\": [\"$i\"], \"force\":true}"; + echo + echo done diff --git a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load index f4924aac4..26d3c73e1 100755 --- a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load +++ b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load @@ -21,58 +21,76 @@ echo echo "Setting up default Security Onion package policies for Elastic Agent..." # Set up Suricata logs +echo echo "Setting up Suricata package policy..." curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "suricata-logs", "name": "suricata-logs", "description": "Suricata integration", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/suricata/eve*.json" ], "data_stream.dataset": "suricata", "tags": [],"processors": "- add_fields:\n target: event\n fields:\n category: network\n module: suricata", "custom": "pipeline: suricata.common" }}}}}}' +echo # Set up Zeek logs +echo echo "Setting up Zeek package policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "name": "zeek-logs", "description": "Zeek logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": ["/nsm/zeek/logs/current/*.log"], "data_stream.dataset": "zeek", "tags": [], "processors": "- dissect:\n tokenizer: \"/nsm/zeek/logs/current/%{pipeline}.log\"\n field: \"log.file.path\"\n trim_chars: \".log\"\n target_prefix: \"\"\n- script:\n lang: javascript\n source: >\n function process(event) {\n var pl = event.Get(\"pipeline\");\n event.Put(\"@metadata.pipeline\", \"zeek.\" + pl);\n }\n- add_fields:\n target: event\n fields:\n category: network\n module: zeek", "custom": "prospector.scanner.exclude_files: [\"(broker | capture_loss | loaded_scripts | packet_filter | stats | stderr | stdout).log$\"]\n" } } } } } }' +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "name": "zeek-logs", "description": "Zeek logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": ["/nsm/zeek/logs/current/*.log"], "data_stream.dataset": "zeek", "tags": [], "processors": "- dissect:\n tokenizer: \"/nsm/zeek/logs/current/%{pipeline}.log\"\n field: \"log.file.path\"\n trim_chars: \".log\"\n target_prefix: \"\"\n- script:\n lang: javascript\n source: >\n function process(event) {\n var pl = event.Get(\"pipeline\");\n event.Put(\"@metadata.pipeline\", \"zeek.\" + pl);\n }\n- add_fields:\n target: event\n fields:\n category: network\n module: zeek\n- add_tags:\n tags: \"ics\"\n when:\n regexp:\n pipeline: \"^bacnet*|^bsap*|^cip*|^cotp*|^dnp3*|^ecat*|^enip*|^modbus*|^opcua*|^profinet*|^s7comm*\"", "custom": "prospector.scanner.exclude_files: [\"(broker | capture_loss | loaded_scripts | packet_filter | stats | stderr | stdout).log$\"]\n" } } } } } }' +echo # Import - Suricata +echo echo "Settings up Suricata import package policy..." curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "import-suricata-logs", "name": "import-suricata-logs", "description": "Import Suricata logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": ["/nsm/import/*/suricata/eve*.json"], "data_stream.dataset": "import", "tags": ["import"], "processors": "- add_fields:\n target: event\n fields:\n category: file\n module: suricata\n imported: true\n- dissect:\n tokenizer: \"/nsm/import/%{import.id}/suricata/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"", "custom": "pipeline: suricata.common" } } } } } }' +echo # Set Import - Zeek logs +echo echo "Setting up Zeek Import package policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "name": "import-zeek-logs", "description": "Zeek Import logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": ["/nsm/import/*/zeek/logs/*.log"], "data_stream.dataset": "import", "tags": [], "processors": "- dissect:\n tokenizer: \"/nsm/import/%{import.id}/zeek/logs/%{pipeline}.log\"\n field: \"log.file.path\"\n trim_chars: \".log\"\n target_prefix: \"\"\n- script:\n lang: javascript\n source: >\n function process(event) {\n var pl = event.Get(\"pipeline\");\n event.Put(\"@metadata.pipeline\", \"zeek.\" + pl);\n }\n- add_fields:\n target: event\n fields:\n category: network\n module: zeek\n imported: true", "custom": "prospector.scanner.exclude_files: [\"(broker | capture_loss | loaded_scripts | packet_filter | stats | stderr | stdout).log$\"]" } } } } } }' - +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "name": "import-zeek-logs", "description": "Zeek Import logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": ["/nsm/import/*/zeek/logs/*.log"], "data_stream.dataset": "import", "tags": [], "processors": "- dissect:\n tokenizer: \"/nsm/import/%{import.id}/zeek/logs/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"\n- script:\n lang: javascript\n source: >\n function process(event) {\n var pl = event.Get(\"import.file\").slice(0,-4);\n event.Put(\"@metadata.pipeline\", \"zeek.\" + pl);\n }\n- add_fields:\n target: event\n fields:\n category: network\n module: zeek\n imported: true\n- add_tags:\n tags: \"ics\"\n when:\n regexp:\n pipeline: \"^bacnet*|^bsap*|^cip*|^cotp*|^dnp3*|^ecat*|^enip*|^modbus*|^opcua*|^profinet*|^s7comm*\"" } } } } } }' # Strelka logs echo "Setting up Strelka package policy" curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "strelka-logs", "name": "strelka-logs", "description": "Strelka logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/strelka/log/strelka.log" ], "data_stream.dataset": "file", "tags": [],"processors": "- add_fields:\n target: event\n fields:\n category: file\n module: strelka", "custom": "pipeline: strelka.file" }}}}}}' +echo # Syslog TCP Port 514 +echo echo "Setting up Syslog TCP package policy..." curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{ "policy_id": "so-grid-nodes", "package": { "name": "tcp", "version": "1.5.0" }, "id": "syslog-tcp-514", "name": "syslog-tcp-514", "description": "Syslog Over TCP Port 514", "namespace": "so", "inputs": { "tcp-tcp": { "enabled": true, "streams": { "tcp.generic": { "enabled": true, "vars": { "listen_address": "0.0.0.0", "listen_port": "514", "data_stream.dataset": "syslog", "pipeline": "syslog", "processors": "- add_fields:\n target: event\n fields:\n module: syslog", "tags": [ "syslog" ], "syslog_options": "field: message\n#format: auto\n#timezone: Local" } } } } } }' +echo # Syslog UDP Port 514 +echo echo "Setting up Syslog UDP package policy..." curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{ "policy_id": "so-grid-nodes", "package": { "name": "udp", "version": "1.5.0" }, "id": "syslog-udp-514", "name": "syslog-udp-514", "description": "Syslog over UDP Port 514", "namespace": "so", "inputs": { "udp-udp": { "enabled": true, "streams": { "udp.generic": { "enabled": true, "vars": { "listen_address": "0.0.0.0", "listen_port": "514", "data_stream.dataset": "syslog", "pipeline": "syslog", "max_message_size": "10KiB", "keep_null": false, "processors": "- add_fields:\n target: event\n fields: \n module: syslog\n", "tags": [ "syslog" ], "syslog_options": "field: message\n#format: auto\n#timezone: Local" } } } } } }' +echo # Kratos logs +echo echo "Setting up Kratos package policy..." curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "kratos-logs", "name": "kratos-logs", "description": "Kratos logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/opt/so/log/kratos/kratos.log" ], "data_stream.dataset": "access", "tags": [],"processors": "- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n add_error_key: true\n- rename:\n fields:\n - from: \"audience\"\n to: \"event.dataset\"\n ignore_missing: true\n- add_fields:\n when:\n not: \n has_fields: ['event.dataset']\n target: ''\n fields:\n event.dataset: access", "custom": "pipeline: kratos" }}}}}}' +echo -# RITA Beacon logs -echo "Setting up RITA Beacon package policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "rita-beacon-logs", "name": "rita-beacon-logs", "description": "RITA Beacon logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/rita/beacons.csv" ], "data_stream.dataset": "beacon", "tags": [],"processors": "- add_fields:\n target: event\n fields:\n category: network\n module: rita", "custom": "pipeline: rita.beacon" }}}}}}' - -# RITA Connection Logs -echo "Setting up RITA connection package policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "rita-connection-logs", "name": "rita-connection-logs", "description": "RITA Connection logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/rita/long-connections.csv" ], "data_stream.dataset": "connections", "tags": [],"processors": "- add_fields:\n target: event\n fields:\n category: network\n module: rita", "custom": "pipeline: rita.connections" }}}}}}' - -# RITA DNS Logs -echo "Setting up RITA DNS package policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "rita-exploded-dns-logs", "name": "rita-exploded-dns-logs", "description": "RITA Exploded DNS logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/rita/exploded-dns.csv" ], "data_stream.dataset": "exploded_dns", "tags": [],"processors": "- add_fields:\n target: event\n fields:\n category: network\n module: rita", "custom": "pipeline: rita.dns" }}}}}}' +# RITA Logs +echo +echo "Setting up RITA package policy..." +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "rita-logs", "name": "rita-logs", "description": "RITA Beacon logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/rita/beacons.csv", "/nsm/rita/long-connections.csv", "/nsm/rita/short-connections.csv", "/nsm/rita/exploded-dns.csv" ], "data_stream.dataset": "rita", "tags": [], "processors": "- add_fields:\n target: event\n fields:\n category: network\n module: rita\n- if:\n log.file.path: beacons.csv\n then: \n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: rita.beacon\n- if:\n regexp:\n log.file.path: \"*connections.csv\"\n then: \n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: rita.connection\n- if:\n log.file.path: \"exploded-dns.csv\"\n then: \n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: rita.dns" }}}}}}' +echo # Elasticsearch logs +echo +echo "Seting up Elasticsearch package policy..." curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "elasticsearch", "version": "1.0.0" }, "id": "elasticsearch-logs", "name": "elasticsearch-logs", "description": "Elasticsearch Logs", "namespace": "default", "inputs": { "elasticsearch-logfile": { "enabled": true, "streams": { "elasticsearch.audit": { "enabled": false, "vars": { "paths": [ "/var/log/elasticsearch/*_audit.json" ] } }, "elasticsearch.deprecation": { "enabled": false, "vars": { "paths": [ "/var/log/elasticsearch/*_deprecation.json" ] } }, "elasticsearch.gc": { "enabled": false, "vars": { "paths": [ "/var/log/elasticsearch/gc.log.[0-9]*", "/var/log/elasticsearch/gc.log" ] } }, "elasticsearch.server": { "enabled": true, "vars": { "paths": [ "/opt/so/log/elasticsearch/*.log" ] } }, "elasticsearch.slowlog": { "enabled": false, "vars": { "paths": [ "/var/log/elasticsearch/*_index_search_slowlog.json", "/var/log/elasticsearch/*_index_indexing_slowlog.json" ] } } } }, "elasticsearch-elasticsearch/metrics": { "enabled": false, "vars": { "hosts": [ "http://localhost:9200" ], "scope": "node" }, "streams": { "elasticsearch.stack_monitoring.ccr": { "enabled": false }, "elasticsearch.stack_monitoring.cluster_stats": { "enabled": false }, "elasticsearch.stack_monitoring.enrich": { "enabled": false }, "elasticsearch.stack_monitoring.index": { "enabled": false }, "elasticsearch.stack_monitoring.index_recovery": { "enabled": false, "vars": { "active.only": true } }, "elasticsearch.stack_monitoring.index_summary": { "enabled": false }, "elasticsearch.stack_monitoring.ml_job": { "enabled": false }, "elasticsearch.stack_monitoring.node": { "enabled": false }, "elasticsearch.stack_monitoring.node_stats": { "enabled": false }, "elasticsearch.stack_monitoring.pending_tasks": { "enabled": false }, "elasticsearch.stack_monitoring.shard": { "enabled": false } } } } }' +echo # Logstash logs +echo +echo "Setting up Logstash package policy..." curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "logstash", "version": "2.0.0" }, "id": "logstash-logs", "name": "logstash-logs", "description": "Logstash logs", "namespace": "default", "inputs": { "logstash-logfile": { "enabled": true, "streams": { "logstash.log": { "enabled": true, "vars": { "paths": [ "/opt/so/logs/logstash/logstash.log" ] } }, "logstash.slowlog": { "enabled": false, "vars": { "paths": [ "/var/log/logstash/logstash-slowlog-plain*.log", "/var/log/logstash/logstash-slowlog-json*.log" ] } } } }, "logstash-logstash/metrics": { "enabled": false, "vars": { "hosts": [ "http://localhost:9600" ], "period": "10s" }, "streams": { "logstash.stack_monitoring.node": { "enabled": false }, "logstash.stack_monitoring.node_stats": { "enabled": false } } } } }' +echo # Kibana logs +echo +echo "Setting up Kibana package policy..." curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "kibana", "version": "2.0.0" }, "id": "kibana-logs", "name": "kibana-logs", "description": "Kibana logs", "namespace": "default", "inputs": { "kibana-logfile": { "enabled": true, "streams": { "kibana.audit": { "enabled": false, "vars": { "paths": [ "/opt/so/log/kibana/kibana.log" ] } }, "kibana.log": { "enabled": true, "vars": { "paths": [ "/opt/so/log/kibana/kibana.log" ] } } } }, "kibana-kibana/metrics": { "enabled": false, "vars": { "hosts": [ "http://localhost:5601" ] }, "streams": { "kibana.stack_monitoring.cluster_actions": { "enabled": false }, "kibana.stack_monitoring.cluster_rules": { "enabled": false }, "kibana.stack_monitoring.node_actions": { "enabled": false }, "kibana.stack_monitoring.node_rules": { "enabled": false }, "kibana.stack_monitoring.stats": { "enabled": false }, "kibana.stack_monitoring.status": { "enabled": false } } } } }' +echo # Redis logs +echo +echo "Setting up Redis package policy..." curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "redis", "version": "1.4.0" }, "id": "redis-logs", "name": "redis-logs", "description": "Redis logs", "namespace": "default", "inputs": { "redis-logfile": { "enabled": true, "streams": { "redis.log": { "enabled": true, "vars": { "paths": [ "/opt/so/log/redis/redis.log" ], "tags": [ "redis-log" ], "preserve_original_event": false } } } }, "redis-redis": { "enabled": false, "streams": { "redis.slowlog": { "enabled": false, "vars": { "hosts": [ "127.0.0.1:6379" ], "password": "" } } } }, "redis-redis/metrics": { "enabled": false, "vars": { "hosts": [ "127.0.0.1:6379" ], "idle_timeout": "20s", "maxconn": 10, "network": "tcp", "password": "" }, "streams": { "redis.info": { "enabled": false, "vars": { "period": "10s" } }, "redis.key": { "enabled": false, "vars": { "key.patterns": "- limit: 20\n pattern: '*'\n", "period": "10s" } }, "redis.keyspace": { "enabled": false, "vars": { "period": "10s" } } } } } }' - +echo From 44d149b1c3c1ebb6018306ca80ee814ac1ce9dc1 Mon Sep 17 00:00:00 2001 From: Wes Date: Tue, 24 Jan 2023 17:01:52 +0000 Subject: [PATCH 06/10] Allow imported data to use a tag of 'import' --- .../pipelines/config/so/9805_output_elastic_agent.conf.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/logstash/pipelines/config/so/9805_output_elastic_agent.conf.jinja b/salt/logstash/pipelines/config/so/9805_output_elastic_agent.conf.jinja index 1c770c7b0..0a148155c 100644 --- a/salt/logstash/pipelines/config/so/9805_output_elastic_agent.conf.jinja +++ b/salt/logstash/pipelines/config/so/9805_output_elastic_agent.conf.jinja @@ -1,5 +1,5 @@ output { - if "elastic-agent" in [tags] and "import" not in [tags] { + if "elastic-agent" in [tags] { if [metadata][pipeline] { elasticsearch { hosts => "{{ GLOBALS.manager }}" From 38ead7cb8264bd3b30b31490c28b01c8fa12151c Mon Sep 17 00:00:00 2001 From: Wes Date: Tue, 24 Jan 2023 17:58:19 +0000 Subject: [PATCH 07/10] Remove import tag for now --- salt/common/tools/sbin/so-elastic-fleet-integration-policy-load | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load index 26d3c73e1..aa31e9fb6 100755 --- a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load +++ b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load @@ -35,7 +35,7 @@ echo # Import - Suricata echo echo "Settings up Suricata import package policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "import-suricata-logs", "name": "import-suricata-logs", "description": "Import Suricata logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": ["/nsm/import/*/suricata/eve*.json"], "data_stream.dataset": "import", "tags": ["import"], "processors": "- add_fields:\n target: event\n fields:\n category: file\n module: suricata\n imported: true\n- dissect:\n tokenizer: \"/nsm/import/%{import.id}/suricata/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"", "custom": "pipeline: suricata.common" } } } } } }' +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "import-suricata-logs", "name": "import-suricata-logs", "description": "Import Suricata logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": ["/nsm/import/*/suricata/eve*.json"], "data_stream.dataset": "import", "tags": [], "processors": "- add_fields:\n target: event\n fields:\n category: file\n module: suricata\n imported: true\n- dissect:\n tokenizer: \"/nsm/import/%{import.id}/suricata/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"", "custom": "pipeline: suricata.common" } } } } } }' echo # Set Import - Zeek logs From 4b9c92c53d91f0dd2639d68228b028e18dd95ce0 Mon Sep 17 00:00:00 2001 From: Wes Date: Tue, 24 Jan 2023 18:00:34 +0000 Subject: [PATCH 08/10] Set RITA event.dataset value explicitly --- salt/elasticsearch/files/ingest/rita.beacon | 4 ++-- salt/elasticsearch/files/ingest/rita.connection | 4 ++-- salt/elasticsearch/files/ingest/rita.dns | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/salt/elasticsearch/files/ingest/rita.beacon b/salt/elasticsearch/files/ingest/rita.beacon index 2c6138262..81bd34e9d 100644 --- a/salt/elasticsearch/files/ingest/rita.beacon +++ b/salt/elasticsearch/files/ingest/rita.beacon @@ -3,8 +3,8 @@ "processors": [ { "set": { - "field": "_index", - "value": "so-rita", + "field": "event.dataset", + "value": "beacon", "override": true } }, diff --git a/salt/elasticsearch/files/ingest/rita.connection b/salt/elasticsearch/files/ingest/rita.connection index 58cc921f4..2c0e33c87 100644 --- a/salt/elasticsearch/files/ingest/rita.connection +++ b/salt/elasticsearch/files/ingest/rita.connection @@ -3,8 +3,8 @@ "processors": [ { "set": { - "field": "_index", - "value": "so-rita", + "field": "event.dataset", + "value": "connection", "override": true } }, diff --git a/salt/elasticsearch/files/ingest/rita.dns b/salt/elasticsearch/files/ingest/rita.dns index 7583bc320..837571b70 100644 --- a/salt/elasticsearch/files/ingest/rita.dns +++ b/salt/elasticsearch/files/ingest/rita.dns @@ -3,8 +3,8 @@ "processors": [ { "set": { - "field": "_index", - "value": "so-rita", + "field": "event.dataset", + "value": "dns", "override": true } }, From 7bf9d77962768af01e981ec491e3f4e9fbe318df Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 25 Jan 2023 08:18:21 -0500 Subject: [PATCH 09/10] Rename Kratos data stream --- salt/common/tools/sbin/so-elastic-fleet-integration-policy-load | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load index aa31e9fb6..b3148f7aa 100755 --- a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load +++ b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load @@ -62,7 +62,7 @@ echo # Kratos logs echo echo "Setting up Kratos package policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "kratos-logs", "name": "kratos-logs", "description": "Kratos logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/opt/so/log/kratos/kratos.log" ], "data_stream.dataset": "access", "tags": [],"processors": "- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n add_error_key: true\n- rename:\n fields:\n - from: \"audience\"\n to: \"event.dataset\"\n ignore_missing: true\n- add_fields:\n when:\n not: \n has_fields: ['event.dataset']\n target: ''\n fields:\n event.dataset: access", "custom": "pipeline: kratos" }}}}}}' +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "kratos-logs", "name": "kratos-logs", "description": "Kratos logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/opt/so/log/kratos/kratos.log" ], "data_stream.dataset": "kratos", "tags": [],"processors": "- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n add_error_key: true\n- rename:\n fields:\n - from: \"audience\"\n to: \"event.dataset\"\n ignore_missing: true\n- add_fields:\n when:\n not: \n has_fields: ['event.dataset']\n target: ''\n fields:\n event.dataset: access", "custom": "pipeline: kratos" }}}}}}' echo # RITA Logs From c9f458e1e2d7dd1c4fa1e5b24b356ea6c28d4cba Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 25 Jan 2023 08:19:50 -0500 Subject: [PATCH 10/10] Set event.dataset for all Kratos logs to 'access' for now --- salt/elasticsearch/files/ingest/kratos | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/elasticsearch/files/ingest/kratos b/salt/elasticsearch/files/ingest/kratos index c55ebd60a..f8dcf53e0 100644 --- a/salt/elasticsearch/files/ingest/kratos +++ b/salt/elasticsearch/files/ingest/kratos @@ -1,6 +1,7 @@ { "description" : "kratos", "processors" : [ + { "set": { "field": "event.dataset", "value": "access" } }, { "pipeline": { "name": "common" } } ] }