From a146f1134e57cef9e7e2a2d8f9c8339716dccd36 Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 11 Jan 2023 13:54:42 +0000 Subject: [PATCH 1/8] Add Elastic Agent utility scripts --- .../sbin/so-elastic-fleet-agent-policy-delete | 14 ++++ .../sbin/so-elastic-fleet-agent-policy-list | 12 +++ .../sbin/so-elastic-fleet-data-streams-list | 12 +++ ...so-elastic-fleet-integration-policy-delete | 14 ++++ .../so-elastic-fleet-integration-policy-list | 12 +++ .../so-elastic-fleet-integration-policy-load | 84 +++++++++++++++++++ salt/common/tools/sbin/so-elastic-fleet-setup | 3 + 7 files changed, 151 insertions(+) create mode 100755 salt/common/tools/sbin/so-elastic-fleet-agent-policy-delete create mode 100755 salt/common/tools/sbin/so-elastic-fleet-agent-policy-list create mode 100755 salt/common/tools/sbin/so-elastic-fleet-data-streams-list create mode 100755 salt/common/tools/sbin/so-elastic-fleet-integration-policy-delete create mode 100755 salt/common/tools/sbin/so-elastic-fleet-integration-policy-list create mode 100755 salt/common/tools/sbin/so-elastic-fleet-integration-policy-load diff --git a/salt/common/tools/sbin/so-elastic-fleet-agent-policy-delete b/salt/common/tools/sbin/so-elastic-fleet-agent-policy-delete new file mode 100755 index 000000000..0564ad3c6 --- /dev/null +++ b/salt/common/tools/sbin/so-elastic-fleet-agent-policy-delete @@ -0,0 +1,14 @@ +#/bin/bash +. /usr/sbin/so-common + +POLICY_ID=$1 + +# Let's snag a cookie from Kibana +SESSIONCOOKIE=$(curl -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}') + +echo "Deleting agent policy $POLICY_ID..." + +# Delete agent policy +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/agent_policies/delete" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d"{\"agentPolicyId\": \"$POLICY_ID\"}" + +echo diff --git a/salt/common/tools/sbin/so-elastic-fleet-agent-policy-list b/salt/common/tools/sbin/so-elastic-fleet-agent-policy-list new file mode 100755 index 000000000..8a9638922 --- /dev/null +++ b/salt/common/tools/sbin/so-elastic-fleet-agent-policy-list @@ -0,0 +1,12 @@ +#/bin/bash +. /usr/sbin/so-common + +# Let's snag a cookie from Kibana +SESSIONCOOKIE=$(curl -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}') + +echo "Setting up default Security Onion package policies for Elastic Agent..." + +# List configured agent policies +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/agent_policies" | jq + +echo diff --git a/salt/common/tools/sbin/so-elastic-fleet-data-streams-list b/salt/common/tools/sbin/so-elastic-fleet-data-streams-list new file mode 100755 index 000000000..9348428e8 --- /dev/null +++ b/salt/common/tools/sbin/so-elastic-fleet-data-streams-list @@ -0,0 +1,12 @@ +#/bin/bash +. /usr/sbin/so-common + +# Let's snag a cookie from Kibana +SESSIONCOOKIE=$(curl -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}') + +echo "Retrieving data stream information..." + +# Retrieve data stream information +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/data_streams" | jq + +echo diff --git a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-delete b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-delete new file mode 100755 index 000000000..b66b7eca8 --- /dev/null +++ b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-delete @@ -0,0 +1,14 @@ +#/bin/bash +. /usr/sbin/so-common + +POLICY_ID=$1 + +# Let's snag a cookie from Kibana +SESSIONCOOKIE=$(curl -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}') + +echo "Deleting integration policy $POLICY_ID..." + +# List configured package policies +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies/delete" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d"{\"packagePolicyIds\": [\"$POLICY_ID\"]}" + +echo diff --git a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-list b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-list new file mode 100755 index 000000000..c73e84174 --- /dev/null +++ b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-list @@ -0,0 +1,12 @@ +#/bin/bash +. /usr/sbin/so-common + +# Let's snag a cookie from Kibana +SESSIONCOOKIE=$(curl -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}') + +echo "Setting up default Security Onion package policies for Elastic Agent..." + +# List configured package policies +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/package_policies" | jq + +echo diff --git a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load new file mode 100755 index 000000000..935d59c05 --- /dev/null +++ b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load @@ -0,0 +1,84 @@ +#/bin/bash + +. /usr/sbin/so-common + +{%- set ZEEKVER = salt['pillar.get']('global:mdengine', '') %} +{%- set STRELKAENABLED = salt['pillar.get']('strelka:enabled', '0') %} +{%- set RITAENABLED = salt['pillar.get']('rita:enabled', False) -%} + +wait_for_web_response "http://localhost:5601/api/spaces/space/default" "default" 300 "curl -K /opt/so/conf/elasticsearch/curl.config" +## This hackery will be removed if using Elastic Auth ## + +# Let's snag a cookie from Kibana +SESSIONCOOKIE=$(curl -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}') + +# Disable certain Features from showing up in the Kibana UI +echo +echo "Setting up default Security Onion package policies for Elastic Agent..." + +# Set up Suricata logs +echo "Setting up Suricata package policy..." +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "suricata-logs", "name": "suricata-logs", "description": "Suricata integration", "namespace": "suricata_so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/suricata/eve*.json" ], "data_stream.dataset": "alert", "tags": [],"processors": "- add_fields:\n target: event\n fields:\n category: network\n module: suricata", "custom": "pipeline: suricata.common" }}}}}}' + +{%- if grains['role'] in ['so-eval', 'so-standalone', 'so-sensor', 'so-helix', 'so-heavynode', 'so-import'] %} + {%- if ZEEKVER != 'SURICATA' %} + {% import_yaml 'filebeat/defaults.yaml' as FBD with context %} + {% set FBCONFIG = salt['pillar.get']('filebeat:zeek_logs_enabled', default=FBD.filebeat, merge=True) %} + {%- for LOGNAME in FBCONFIG.zeek_logs_enabled %} + +# Import - Suricata +echo "Settings up Suricata import package policy..." +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "import-suricata-logs", "name": "import-suricata-logs", "description": "Import Suricata logs", "namespace": "import_so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": ["/nsm/import/*/suricata/eve*.json"], "data_stream.dataset": "alert", "tags": ["import"], "processors": "- add_fields:\n target: event\n fields:\n category: file\n module: suricata\n imported: true\n- dissect:\n tokenizer: \"/nsm/import/%{import.id}/suricata/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"", "custom": "pipeline: suricata.common" } } } } } }' + +# Set up Zeek {{ LOGNAME }} logs +echo "Setting up Zeek {{ LOGNAME }} package policy..." +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "name": "zeek-{{ LOGNAME }}", "description": "Zeek {{ LOGNAME }} logs", "namespace": "zeek_so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/zeek/logs/current/{{ LOGNAME }}.log" ], "data_stream.dataset": "{{ LOGNAME }}", "tags": [], "processors": "- add_fields:\n target: event\n fields:\n category: network\n module: zeek", "custom": "pipeline: zeek.{{ LOGNAME }}" }}}}}}' +echo + +# Import - Zeek +echo "Setting up Zeek import {{ LOGNAME }} package policy..." +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "import-zeek-{{ LOGNAME }}-logs", "name": "import-zeek-{{ LOGNAME }}-logs", "description": "Import Zeek {{ LOGNAME }} logs", "namespace": "import_so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": ["/nsm/import/*/zeek/logs/{{ LOGNAME }}.log"], "data_stream.dataset": "{{ LOGNAME }}", "tags": ["import"], "- add_fields:\n target: event\n fields:\n category: file\n module: zeek\n imported: true\n- dissect:\n tokenizer: \"/nsm/import/%{import.id}/zeek/logs/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"", "custom": "pipeline: zeek.{{ LOGNAME }}" } } } } } }' + + {%- endfor %} + {%- endif %} +{%- endif %} + +# Strelka logs +echo "Setting up Strelka package policy" +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "strelka-logs", "name": "strelka-logs", "description": "Strelka logs", "namespace": "strelka_so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/strelka/log/strelka.log" ], "data_stream.dataset": "file", "tags": [],"processors": "- add_fields:\n target: event\n fields:\n category: file\n module: strelka", "custom": "pipeline: strelka.file" }}}}}}' + +# Syslog TCP Port 514 +echo "Setting up Syslog TCP package policy..." +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{ "policy_id": "so-grid-nodes", "package": { "name": "tcp", "version": "1.5.0" }, "id": "syslog-tcp-514", "name": "syslog-tcp-514", "description": "Syslog Over TCP Port 514", "namespace": "syslog_so", "inputs": { "tcp-tcp": { "enabled": true, "streams": { "tcp.generic": { "enabled": true, "vars": { "listen_address": "0.0.0.0", "listen_port": "514", "data_stream.dataset": "syslog", "pipeline": "syslog", "processors": "- add_fields:\n target: event\n fields:\n module: syslog", "tags": [ "syslog" ], "syslog_options": "field: message\n#format: auto\n#timezone: Local\n" } } } } } }' + +# Syslog UDP Port 514 +echo "Setting up Syslog UDP package policy..." +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{ "policy_id": "so-grid-nodes", "package": { "name": "udp", "version": "1.5.0" }, "id": "syslog-udp-514", "name": "syslog-udp-514", "description": "Syslog over UDP Port 514", "namespace": "syslog_so", "inputs": { "udp-udp": { "enabled": true, "streams": { "udp.generic": { "enabled": true, "vars": { "listen_address": "0.0.0.0", "listen_port": "514", "data_stream.dataset": "syslog", "pipeline": "syslog", "max_message_size": "10KiB", "keep_null": false, "processors": "- add_fields:\n target: event\n fields: \n module: syslog\n", "tags": [ "syslog" ], "syslog_options": "field: message\n#format: auto\n#timezone: Local\n" } } } } } }' + +# Kratos logs +echo "Setting up Kratos package policy..." +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "kratos-logs", "name": "kratos-logs", "description": "Kratos logs", "namespace": "kratos_so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/opt/so/log/kratos/kratos.log" ], "data_stream.dataset": "access", "tags": [],"processors": "- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n add_error_key: true\n- rename:\n fields:\n - from: \"audience\"\n to: \"event.dataset\"\n ignore_missing: true\n- add_fields:\n when:\n not: \n has_fields: ['event.dataset']\n target: ''\n fields:\n event.dataset: access", "custom": "pipeline: kratos" }}}}}}' + +# RITA Beacon logs +echo "Setting up RITA Beacon package policy..." +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "rita-beacon-logs", "name": "rita-beacon-logs", "description": "RITA Beacon logs", "namespace": "rita_so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/rita/beacons.csv" ], "data_stream.dataset": "beacon", "tags": [],"processors": "- add_fields:\n target: event\n fields:\n category: network\n module: rita", "custom": "pipeline: rita.beacon" }}}}}}' + +# RITA Connection Logs +echo "Setting up RITA connection package policy..." +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "rita-connection-logs", "name": "rita-connection-logs", "description": "RITA Connection logs", "namespace": "rita_so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/rita/long-connections.csv" ], "data_stream.dataset": "connections", "tags": [],"processors": "- add_fields:\n target: event\n fields:\n category: network\n module: rita", "custom": "pipeline: rita.connections" }}}}}}' + +# RITA DNS Logs +echo "Setting up RITA DNS package policy..." +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "rita-exploded-dns-logs", "name": "rita-exploded-dns-logs", "description": "RITA Exploded DNS logs", "namespace": "rita_so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/rita/exploded-dns.csv" ], "data_stream.dataset": "exploded_dns", "tags": [],"processors": "- add_fields:\n target: event\n fields:\n category: network\n module: rita", "custom": "pipeline: rita.dns" }}}}}}' + +# Elasticsearch logs +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "elasticsearch", "version": "1.0.0" }, "id": "elasticsearch-logs", "name": "elasticsearch-logs", "description": "Elasticsearch Logs", "namespace": "default", "inputs": { "elasticsearch-logfile": { "enabled": true, "streams": { "elasticsearch.audit": { "enabled": false, "vars": { "paths": [ "/var/log/elasticsearch/*_audit.json" ] } }, "elasticsearch.deprecation": { "enabled": false, "vars": { "paths": [ "/var/log/elasticsearch/*_deprecation.json" ] } }, "elasticsearch.gc": { "enabled": false, "vars": { "paths": [ "/var/log/elasticsearch/gc.log.[0-9]*", "/var/log/elasticsearch/gc.log" ] } }, "elasticsearch.server": { "enabled": true, "vars": { "paths": [ "/opt/so/log/elasticsearch/*.log" ] } }, "elasticsearch.slowlog": { "enabled": false, "vars": { "paths": [ "/var/log/elasticsearch/*_index_search_slowlog.json", "/var/log/elasticsearch/*_index_indexing_slowlog.json" ] } } } }, "elasticsearch-elasticsearch/metrics": { "enabled": false, "vars": { "hosts": [ "http://localhost:9200" ], "scope": "node" }, "streams": { "elasticsearch.stack_monitoring.ccr": { "enabled": false }, "elasticsearch.stack_monitoring.cluster_stats": { "enabled": false }, "elasticsearch.stack_monitoring.enrich": { "enabled": false }, "elasticsearch.stack_monitoring.index": { "enabled": false }, "elasticsearch.stack_monitoring.index_recovery": { "enabled": false, "vars": { "active.only": true } }, "elasticsearch.stack_monitoring.index_summary": { "enabled": false }, "elasticsearch.stack_monitoring.ml_job": { "enabled": false }, "elasticsearch.stack_monitoring.node": { "enabled": false }, "elasticsearch.stack_monitoring.node_stats": { "enabled": false }, "elasticsearch.stack_monitoring.pending_tasks": { "enabled": false }, "elasticsearch.stack_monitoring.shard": { "enabled": false } } } } }' + +# Logstash logs +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "logstash", "version": "2.0.0" }, "id": "logstash-logs", "name": "logstash-logs", "description": "Logstash logs", "namespace": "default", "inputs": { "logstash-logfile": { "enabled": true, "streams": { "logstash.log": { "enabled": true, "vars": { "paths": [ "/opt/so/logs/logstash/logstash.log" ] } }, "logstash.slowlog": { "enabled": false, "vars": { "paths": [ "/var/log/logstash/logstash-slowlog-plain*.log", "/var/log/logstash/logstash-slowlog-json*.log" ] } } } }, "logstash-logstash/metrics": { "enabled": false, "vars": { "hosts": [ "http://localhost:9600" ], "period": "10s" }, "streams": { "logstash.stack_monitoring.node": { "enabled": false }, "logstash.stack_monitoring.node_stats": { "enabled": false } } } } }' + +# Kibana logs +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "kibana", "version": "2.0.0" }, "id": "kibana-logs", "name": "kibana-logs", "description": "Kibana logs", "namespace": "default", "inputs": { "kibana-logfile": { "enabled": true, "streams": { "kibana.audit": { "enabled": false, "vars": { "paths": [ "/opt/so/log/kibana/kibana.log" ] } }, "kibana.log": { "enabled": true, "vars": { "paths": [ "/opt/so/log/kibana/kibana.log" ] } } } }, "kibana-kibana/metrics": { "enabled": false, "vars": { "hosts": [ "http://localhost:5601" ] }, "streams": { "kibana.stack_monitoring.cluster_actions": { "enabled": false }, "kibana.stack_monitoring.cluster_rules": { "enabled": false }, "kibana.stack_monitoring.node_actions": { "enabled": false }, "kibana.stack_monitoring.node_rules": { "enabled": false }, "kibana.stack_monitoring.stats": { "enabled": false }, "kibana.stack_monitoring.status": { "enabled": false } } } } }' + +# Redis logs +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "redis", "version": "1.4.0" }, "id": "redis-logs", "name": "redis-logs", "description": "Redis logs", "namespace": "default", "inputs": { "redis-logfile": { "enabled": true, "streams": { "redis.log": { "enabled": true, "vars": { "paths": [ "/opt/so/log/redis/redis.log" ], "tags": [ "redis-log" ], "preserve_original_event": false } } } }, "redis-redis": { "enabled": false, "streams": { "redis.slowlog": { "enabled": false, "vars": { "hosts": [ "127.0.0.1:6379" ], "password": "" } } } }, "redis-redis/metrics": { "enabled": false, "vars": { "hosts": [ "127.0.0.1:6379" ], "idle_timeout": "20s", "maxconn": 10, "network": "tcp", "password": "" }, "streams": { "redis.info": { "enabled": false, "vars": { "period": "10s" } }, "redis.key": { "enabled": false, "vars": { "key.patterns": "- limit: 20\n pattern: '*'\n", "period": "10s" } }, "redis.keyspace": { "enabled": false, "vars": { "period": "10s" } } } } } }' diff --git a/salt/common/tools/sbin/so-elastic-fleet-setup b/salt/common/tools/sbin/so-elastic-fleet-setup index bc331c2af..4e3d7639c 100755 --- a/salt/common/tools/sbin/so-elastic-fleet-setup +++ b/salt/common/tools/sbin/so-elastic-fleet-setup @@ -75,6 +75,9 @@ printf '%s\n'\ # Call Elastic-Fleet Salt State salt-call state.apply elastic-fleet queue=True +# Load Elastic Fleet integrations +/usr/sbin/so-elastic-fleet-integration-policy-load + # Temp wget -P /opt/so/saltstack/default/salt/elastic-fleet/files/elastic-agent/ https://github.com/Security-Onion-Solutions/securityonion-docker-rpm/releases/download/so_elastic-agent-8.4.1/so-elastic-agent-8.4.1-darwin-x86_64.tar.gz wget -P /opt/so/saltstack/default/salt/elastic-fleet/files/elastic-agent/ https://github.com/Security-Onion-Solutions/securityonion-docker-rpm/releases/download/so_elastic-agent-8.4.1/so-elastic-agent-8.4.1-linux-x86_64.tar.gz From caf0ea6b5318aa356624113ad8df233c235e4701 Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 11 Jan 2023 13:56:21 +0000 Subject: [PATCH 2/8] Add Elastic Agent policy view script --- .../tools/sbin/so-elastic-fleet-agent-policy-view | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 salt/common/tools/sbin/so-elastic-fleet-agent-policy-view diff --git a/salt/common/tools/sbin/so-elastic-fleet-agent-policy-view b/salt/common/tools/sbin/so-elastic-fleet-agent-policy-view new file mode 100644 index 000000000..6ab6474b0 --- /dev/null +++ b/salt/common/tools/sbin/so-elastic-fleet-agent-policy-view @@ -0,0 +1,14 @@ +#/bin/bash +. /usr/sbin/so-common + +POLICY_ID=$1 + +# Let's snag a cookie from Kibana +SESSIONCOOKIE=$(curl -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}') + +echo "Viewing agent policy $POLICY_ID" + +# View agent policy +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/agent_policies/$POLICY_ID/full" | jq + +echo From 5d86edeed4e1414c0343bc7ed6c3d251b54e33ee Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 11 Jan 2023 13:57:32 +0000 Subject: [PATCH 3/8] Modify Logstash Elastic Agent output to accomodate for events with and without 'metadata.pipeline' --- .../so/9805_output_elastic_agent.conf.jinja | 30 ++++++++++++++----- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/salt/logstash/pipelines/config/so/9805_output_elastic_agent.conf.jinja b/salt/logstash/pipelines/config/so/9805_output_elastic_agent.conf.jinja index ed81d4322..1c770c7b0 100644 --- a/salt/logstash/pipelines/config/so/9805_output_elastic_agent.conf.jinja +++ b/salt/logstash/pipelines/config/so/9805_output_elastic_agent.conf.jinja @@ -1,14 +1,28 @@ output { if "elastic-agent" in [tags] and "import" not in [tags] { - elasticsearch { - hosts => "{{ GLOBALS.manager }}" - ecs_compatibility => v8 - data_stream => true - user => "{{ ES_USER }}" - password => "{{ ES_PASS }}" - ssl => true - ssl_certificate_verification => false + if [metadata][pipeline] { + elasticsearch { + hosts => "{{ GLOBALS.manager }}" + ecs_compatibility => v8 + data_stream => true + user => "{{ ES_USER }}" + password => "{{ ES_PASS }}" + pipeline => "%{[metadata][pipeline]}" + ssl => true + ssl_certificate_verification => false + } } + else { + elasticsearch { + hosts => "{{ GLOBALS.manager }}" + ecs_compatibility => v8 + data_stream => true + user => "{{ ES_USER }}" + password => "{{ ES_PASS }}" + ssl => true + ssl_certificate_verification => false + } + } } } From 2e886d0c55bb6a5a27cfede964d5a840853200ff Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 11 Jan 2023 13:58:38 +0000 Subject: [PATCH 4/8] Remove data_index_name processor since we are using data streams --- salt/elasticsearch/files/ingest-dynamic/common | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/salt/elasticsearch/files/ingest-dynamic/common b/salt/elasticsearch/files/ingest-dynamic/common index bdf6513fe..277a4f14d 100644 --- a/salt/elasticsearch/files/ingest-dynamic/common +++ b/salt/elasticsearch/files/ingest-dynamic/common @@ -57,16 +57,7 @@ { "convert": { "field": "log.id.uid", "type": "string", "ignore_failure": true, "ignore_missing": true } }, { "convert": { "field": "agent.id", "type": "string", "ignore_failure": true, "ignore_missing": true } }, { "convert": { "field": "event.severity", "type": "integer", "ignore_failure": true, "ignore_missing": true } }, - { "remove": { "field": [ "message2", "type", "fields", "category", "module", "dataset" ], "ignore_missing": true, "ignore_failure": true } }, - { - "date_index_name": { - "field": "@timestamp", - "index_name_prefix": "{{ _index }}-", - "date_rounding": "d", - "ignore_failure": true, - "index_name_format": "yyyy.MM.dd" - } - } + { "remove": { "field": [ "message2", "type", "fields", "category", "module", "dataset" ], "ignore_missing": true, "ignore_failure": true } } {%- endraw %} {%- if HIGHLANDER %} , From 5062dd287355a4ec8a2c31c34546946d88730fa6 Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 11 Jan 2023 14:02:09 +0000 Subject: [PATCH 5/8] Suricata Elasticsearch ingest node pipeline changes - set 'alert' dataset --- salt/elasticsearch/files/ingest/suricata.alert | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/elasticsearch/files/ingest/suricata.alert b/salt/elasticsearch/files/ingest/suricata.alert index e372b1645..87d5144ed 100644 --- a/salt/elasticsearch/files/ingest/suricata.alert +++ b/salt/elasticsearch/files/ingest/suricata.alert @@ -1,6 +1,7 @@ { "description" : "suricata.alert", "processors" : [ + { "set": { "field": "tags","value": "alert" }}, { "rename":{ "field": "message2.alert", "target_field": "rule", "ignore_failure": true } }, { "rename":{ "field": "rule.signature", "target_field": "rule.name", "ignore_failure": true } }, { "rename":{ "field": "rule.ref", "target_field": "rule.version", "ignore_failure": true } }, From c3b83f1fc8b453e00486162bd9afb6b26d36580a Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 11 Jan 2023 14:03:11 +0000 Subject: [PATCH 6/8] Update template settings to use data streams --- salt/elasticsearch/defaults.yaml | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index f1a5a7dd5..92c5c6e13 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -1152,8 +1152,9 @@ elasticsearch: delete: 365 index_sorting: False index_template: + data_stream: {} index_patterns: - - so-* + - logs-*_so template: mappings: dynamic_templates: @@ -2154,11 +2155,12 @@ elasticsearch: - common-settings - common-dynamic-mappings priority: 500 - so-ids: + so-suricata: index_sorting: False index_template: + data_stream: {} index_patterns: - - so-ids* + - logs-*-suricata_so template: mappings: dynamic_templates: @@ -2324,8 +2326,9 @@ elasticsearch: so-import: index_sorting: False index_template: + data_stream: {} index_patterns: - - so-import* + - logs-*-import_so template: mappings: dynamic_templates: @@ -4162,8 +4165,9 @@ elasticsearch: so-strelka: index_sorting: False index_template: + data_stream: {} index_patterns: - - so-strelka* + - logs-*-strelka_so template: mappings: dynamic_templates: @@ -4415,8 +4419,9 @@ elasticsearch: so-zeek: index_sorting: False index_template: + data_stream: {} index_patterns: - - so-zeek* + - logs-*-zeek_so template: mappings: dynamic_templates: From 33e2affb1d14bfe7979d18e70d885f96e728ab5b Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 11 Jan 2023 14:08:28 +0000 Subject: [PATCH 7/8] Remove newlines from end of Syslog processor definitions --- .../tools/sbin/so-elastic-fleet-integration-policy-load | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load index 935d59c05..5b5e72b76 100755 --- a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load +++ b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load @@ -49,11 +49,11 @@ curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POS # Syslog TCP Port 514 echo "Setting up Syslog TCP package policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{ "policy_id": "so-grid-nodes", "package": { "name": "tcp", "version": "1.5.0" }, "id": "syslog-tcp-514", "name": "syslog-tcp-514", "description": "Syslog Over TCP Port 514", "namespace": "syslog_so", "inputs": { "tcp-tcp": { "enabled": true, "streams": { "tcp.generic": { "enabled": true, "vars": { "listen_address": "0.0.0.0", "listen_port": "514", "data_stream.dataset": "syslog", "pipeline": "syslog", "processors": "- add_fields:\n target: event\n fields:\n module: syslog", "tags": [ "syslog" ], "syslog_options": "field: message\n#format: auto\n#timezone: Local\n" } } } } } }' +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{ "policy_id": "so-grid-nodes", "package": { "name": "tcp", "version": "1.5.0" }, "id": "syslog-tcp-514", "name": "syslog-tcp-514", "description": "Syslog Over TCP Port 514", "namespace": "syslog_so", "inputs": { "tcp-tcp": { "enabled": true, "streams": { "tcp.generic": { "enabled": true, "vars": { "listen_address": "0.0.0.0", "listen_port": "514", "data_stream.dataset": "syslog", "pipeline": "syslog", "processors": "- add_fields:\n target: event\n fields:\n module: syslog", "tags": [ "syslog" ], "syslog_options": "field: message\n#format: auto\n#timezone: Local" } } } } } }' # Syslog UDP Port 514 echo "Setting up Syslog UDP package policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{ "policy_id": "so-grid-nodes", "package": { "name": "udp", "version": "1.5.0" }, "id": "syslog-udp-514", "name": "syslog-udp-514", "description": "Syslog over UDP Port 514", "namespace": "syslog_so", "inputs": { "udp-udp": { "enabled": true, "streams": { "udp.generic": { "enabled": true, "vars": { "listen_address": "0.0.0.0", "listen_port": "514", "data_stream.dataset": "syslog", "pipeline": "syslog", "max_message_size": "10KiB", "keep_null": false, "processors": "- add_fields:\n target: event\n fields: \n module: syslog\n", "tags": [ "syslog" ], "syslog_options": "field: message\n#format: auto\n#timezone: Local\n" } } } } } }' +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{ "policy_id": "so-grid-nodes", "package": { "name": "udp", "version": "1.5.0" }, "id": "syslog-udp-514", "name": "syslog-udp-514", "description": "Syslog over UDP Port 514", "namespace": "syslog_so", "inputs": { "udp-udp": { "enabled": true, "streams": { "udp.generic": { "enabled": true, "vars": { "listen_address": "0.0.0.0", "listen_port": "514", "data_stream.dataset": "syslog", "pipeline": "syslog", "max_message_size": "10KiB", "keep_null": false, "processors": "- add_fields:\n target: event\n fields: \n module: syslog\n", "tags": [ "syslog" ], "syslog_options": "field: message\n#format: auto\n#timezone: Local" } } } } } }' # Kratos logs echo "Setting up Kratos package policy..." From 52b620b137a76bf8970ef0c229cc3151c5c5434c Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 11 Jan 2023 14:10:11 +0000 Subject: [PATCH 8/8] Add additional conditional logic for Filebeat and disable Filebeat --- salt/top.sls | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/salt/top.sls b/salt/top.sls index 3997f9725..ed338a981 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -8,7 +8,7 @@ {% set PLAYBOOK = salt['pillar.get']('manager:playbook', '0') %} {% set ELASTALERT = salt['pillar.get']('elastalert:enabled', True) %} {% set ELASTICSEARCH = salt['pillar.get']('elasticsearch:enabled', True) %} -{% set FILEBEAT = salt['pillar.get']('filebeat:enabled', True) %} +{% set FILEBEAT = salt['pillar.get']('filebeat:enabled', False) %} {% set KIBANA = salt['pillar.get']('kibana:enabled', True) %} {% set LOGSTASH = salt['pillar.get']('logstash:enabled', True) %} {% set REDIS = salt['pillar.get']('redis:enabled', True) %} @@ -62,7 +62,9 @@ base: {%- if STRELKA %} - strelka {%- endif %} + {%- if FILEBEAT %} - filebeat + {%- endif %} - schedule - docker_clean @@ -377,7 +379,9 @@ base: - firewall - schedule - docker_clean + {%- if FILEBEAT %} - filebeat + {%- endif %} - idh 'J@workstation:gui:enabled:^[Tt][Rr][Uu][Ee]$ and ( G@saltversion:{{saltversion}} and G@os:CentOS )':