From 4c4b873eca0ab0901c33d0227aa77b276412f1e6 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 19 Apr 2023 09:04:33 -0400 Subject: [PATCH] Add integrations and cleanup --- salt/common/tools/sbin/so-common | 33 +++++ .../so-elastic-fleet-integration-policy-load | 140 ++---------------- salt/common/tools/sbin/so-elastic-fleet-setup | 54 +------ salt/common/tools/sbin/so-minion | 20 ++- .../endpoints-initial/system-endpoints.json | 76 ++++++++++ .../endpoints-initial/windows-endpoints.json | 59 ++++++++ .../grid-nodes/elasticsearch-logs.json | 106 +++++++++++++ .../grid-nodes/import-evtx-logs.json | 29 ++++ .../grid-nodes/import-suricata-logs.json | 29 ++++ .../grid-nodes/import-zeek-logs.json | 29 ++++ .../grid-nodes/osquery-grid-nodes.json | 20 +++ .../integrations/grid-nodes/redis-logs.json | 76 ++++++++++ .../grid-nodes/soc-auth-sync-logs.json | 29 ++++ .../grid-nodes/soc-salt-relay-logs.json | 29 ++++ .../grid-nodes/soc-sensoroni-logs.json | 29 ++++ .../grid-nodes/soc-server-logs.json | 29 ++++ .../integrations/grid-nodes/strelka-logs.json | 29 ++++ .../grid-nodes/suricata-logs.json | 29 ++++ .../grid-nodes/syslog-tcp-514.json | 32 ++++ .../grid-nodes/syslog-udp-514.json | 33 +++++ .../grid-nodes/system-grid-nodes.json | 47 ++++++ .../integrations/grid-nodes/zeek-logs.json | 29 ++++ salt/top.sls | 1 + setup/so-setup | 2 + 24 files changed, 812 insertions(+), 177 deletions(-) create mode 100644 salt/elasticfleet/files/integrations/endpoints-initial/system-endpoints.json create mode 100644 salt/elasticfleet/files/integrations/endpoints-initial/windows-endpoints.json create mode 100644 salt/elasticfleet/files/integrations/grid-nodes/elasticsearch-logs.json create mode 100644 salt/elasticfleet/files/integrations/grid-nodes/import-evtx-logs.json create mode 100644 salt/elasticfleet/files/integrations/grid-nodes/import-suricata-logs.json create mode 100644 salt/elasticfleet/files/integrations/grid-nodes/import-zeek-logs.json create mode 100644 salt/elasticfleet/files/integrations/grid-nodes/osquery-grid-nodes.json create mode 100644 salt/elasticfleet/files/integrations/grid-nodes/redis-logs.json create mode 100644 salt/elasticfleet/files/integrations/grid-nodes/soc-auth-sync-logs.json create mode 100644 salt/elasticfleet/files/integrations/grid-nodes/soc-salt-relay-logs.json create mode 100644 salt/elasticfleet/files/integrations/grid-nodes/soc-sensoroni-logs.json create mode 100644 salt/elasticfleet/files/integrations/grid-nodes/soc-server-logs.json create mode 100644 salt/elasticfleet/files/integrations/grid-nodes/strelka-logs.json create mode 100644 salt/elasticfleet/files/integrations/grid-nodes/suricata-logs.json create mode 100644 salt/elasticfleet/files/integrations/grid-nodes/syslog-tcp-514.json create mode 100644 salt/elasticfleet/files/integrations/grid-nodes/syslog-udp-514.json create mode 100644 salt/elasticfleet/files/integrations/grid-nodes/system-grid-nodes.json create mode 100644 salt/elasticfleet/files/integrations/grid-nodes/zeek-logs.json diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 06d359748..5089adb64 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -160,6 +160,39 @@ disable_fastestmirror() { sed -i 's/enabled=1/enabled=0/' /etc/yum/pluginconf.d/fastestmirror.conf } +elastic_fleet_integration_create() { + + JSON_STRING=$1 + + curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" +} + +elastic_fleet_policy_create() { + + NAME=$1 + DESC=$2 + FLEETSERVER=$3 + + JSON_STRING=$( jq -n \ + --arg NAME "$NAME" \ + --arg DESC "$DESC" \ + --arg FLEETSERVER "$FLEETSERVER" \ + '{"name": $NAME,"id":$NAME,"description":$DESC,"namespace":"default","monitoring_enabled":["logs"],"inactivity_timeout":1209600,"has_fleet_server":$FLEETSERVER}' + ) + # Create Fleet Policy + curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/agent_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" + +} + +elastic_fleet_policy_update() { + + POLICYID=$1 + JSON_STRING=$2 + + curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/agent_policies/$POLICYID" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" +} + + elastic_license() { read -r -d '' message <<- EOM diff --git a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load index 4efdd5784..a65e29244 100755 --- a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load +++ b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load @@ -6,132 +6,16 @@ . /usr/sbin/so-common -{%- set ZEEKVER = salt['pillar.get']('global:mdengine', '') %} -{%- set STRELKAENABLED = salt['pillar.get']('strelka:enabled', '0') %} -{%- set RITAENABLED = salt['pillar.get']('rita:enabled', False) %} +# Initial Endpoints +for INTEGRATION in /opt/so/saltstack/default/salt/elasticfleet/files/integrations/endpoints-initial/*.json +do + printf "\n\nInitial Endpoint Policy - Loading $INTEGRATION\n" + elastic_fleet_integration_create "@$INTEGRATION" +done -wait_for_web_response "http://localhost:5601/api/spaces/space/default" "default" 300 "curl -K /opt/so/conf/elasticsearch/curl.config" - -# Let's snag a cookie from Kibana -SESSIONCOOKIE=$(curl -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}') - -# Disable certain Features from showing up in the Kibana UI -echo -echo "Disable certain Features from showing up in the Kibana UI" -so-kibana-space-defaults -echo - -# Suricata logs -echo -echo "Setting up Suricata package policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "suricata-logs", "name": "suricata-logs", "description": "Suricata integration", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/suricata/eve*.json" ], "data_stream.dataset": "suricata", "tags": [],"processors": "- add_fields:\n target: event\n fields:\n category: network\n module: suricata", "custom": "pipeline: suricata.common" }}}}}}' -echo - -# Zeek logs -echo -echo "Setting up Zeek package policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "name": "zeek-logs", "description": "Zeek logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": ["/nsm/zeek/logs/current/*.log"], "data_stream.dataset": "zeek", "tags": [], "processors": "- dissect:\n tokenizer: \"/nsm/zeek/logs/current/%{pipeline}.log\"\n field: \"log.file.path\"\n trim_chars: \".log\"\n target_prefix: \"\"\n- script:\n lang: javascript\n source: >\n function process(event) {\n var pl = event.Get(\"pipeline\");\n event.Put(\"@metadata.pipeline\", \"zeek.\" + pl);\n }\n- add_fields:\n target: event\n fields:\n category: network\n module: zeek\n- add_tags:\n tags: \"ics\"\n when:\n regexp:\n pipeline: \"^bacnet*|^bsap*|^cip*|^cotp*|^dnp3*|^ecat*|^enip*|^modbus*|^opcua*|^profinet*|^s7comm*\"", "custom": "exclude_files: [\"broker|capture_loss|ecat_arp_info|loaded_scripts|packet_filter|stats|stderr|stdout.log$\"]\n" } } } } } }' -echo - - -# Import - EVTX -echo -echo "Setting up EVTX import package policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{ "package": { "name": "log", "version": "1.1.0" }, "name": "import-evtx-logs", "namespace": "so", "description": "Import Windows EVTX logs", "policy_id": "so-grid-nodes", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/import/*/evtx/data.json" ], "data_stream.dataset": "import", "custom": "pipeline: import.wel", "processors": "- dissect:\n tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"\n- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n- add_fields:\n target: event\n fields:\n module: windows_eventlog\n imported: true", "tags": [] } } } } } }' -echo - -# Import - Suricata logs -echo -echo "Setting up Suricata import package policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "import-suricata-logs", "name": "import-suricata-logs", "description": "Import Suricata logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": ["/nsm/import/*/suricata/eve*.json"], "data_stream.dataset": "import", "tags": [], "processors": "- add_fields:\n target: event\n fields:\n category: network\n module: suricata\n imported: true\n- dissect:\n tokenizer: \"/nsm/import/%{import.id}/suricata/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"", "custom": "pipeline: suricata.common" } } } } } }' -echo - -# Import - Zeek logs -echo -echo "Setting up Zeek import package policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "name": "import-zeek-logs", "description": "Zeek Import logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": ["/nsm/import/*/zeek/logs/*.log"], "data_stream.dataset": "import", "tags": [], "processors": "- dissect:\n tokenizer: \"/nsm/import/%{import.id}/zeek/logs/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"\n- script:\n lang: javascript\n source: >\n function process(event) {\n var pl = event.Get(\"import.file\").slice(0,-4);\n event.Put(\"@metadata.pipeline\", \"zeek.\" + pl);\n }\n- add_fields:\n target: event\n fields:\n category: network\n module: zeek\n imported: true\n- add_tags:\n tags: \"ics\"\n when:\n regexp:\n import.file: \"^bacnet*|^bsap*|^cip*|^cotp*|^dnp3*|^ecat*|^enip*|^modbus*|^opcua*|^profinet*|^s7comm*\"", "custom": "exclude_files: [\"broker|capture_loss|ecat_arp_info|loaded_scripts|packet_filter|stats|stderr|stdout.log$\"]\n" } } } } } }' -echo - -# Strelka logs -echo -echo "Setting up Strelka package policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "strelka-logs", "name": "strelka-logs", "description": "Strelka logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/strelka/log/strelka.log" ], "data_stream.dataset": "strelka", "tags": [],"processors": "- add_fields:\n target: event\n fields:\n category: file\n module: strelka", "custom": "pipeline: strelka.file" }}}}}}' -echo - -# Syslog TCP Port 514 -echo -echo "Setting up Syslog TCP package policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{ "policy_id": "so-grid-nodes", "package": { "name": "tcp", "version": "1.5.0" }, "id": "syslog-tcp-514", "name": "syslog-tcp-514", "description": "Syslog Over TCP Port 514", "namespace": "so", "inputs": { "tcp-tcp": { "enabled": true, "streams": { "tcp.generic": { "enabled": true, "vars": { "listen_address": "0.0.0.0", "listen_port": "514", "data_stream.dataset": "syslog", "pipeline": "syslog", "processors": "- add_fields:\n target: event\n fields:\n module: syslog", "tags": [ "syslog" ], "syslog_options": "field: message\n#format: auto\n#timezone: Local" } } } } } }' -echo - -# Syslog UDP Port 514 -echo -echo "Setting up Syslog UDP package policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{ "policy_id": "so-grid-nodes", "package": { "name": "udp", "version": "1.5.0" }, "id": "syslog-udp-514", "name": "syslog-udp-514", "description": "Syslog over UDP Port 514", "namespace": "so", "inputs": { "udp-udp": { "enabled": true, "streams": { "udp.generic": { "enabled": true, "vars": { "listen_address": "0.0.0.0", "listen_port": "514", "data_stream.dataset": "syslog", "pipeline": "syslog", "max_message_size": "10KiB", "keep_null": false, "processors": "- add_fields:\n target: event\n fields: \n module: syslog\n", "tags": [ "syslog" ], "syslog_options": "field: message\n#format: auto\n#timezone: Local" } } } } } }' -echo - -# Kratos logs -echo -echo "Setting up Kratos package policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "kratos-logs", "name": "kratos-logs", "description": "Kratos logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/opt/so/log/kratos/kratos.log" ], "data_stream.dataset": "kratos", "tags": [],"custom":"pipeline: kratos","processors": "- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n add_error_key: true \n- add_fields:\n target: event\n fields:\n category: iam\n module: kratos" }}}}}}' -echo - -# RITA Logs -#echo -#echo "Setting up RITA package policy..." -#curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "rita-logs", "name": "rita-logs", "description": "RITA Beacon logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/rita/beacons.csv", "/nsm/rita/long-connections.csv", "/nsm/rita/short-connections.csv", "/nsm/rita/exploded-dns.csv" ], "data_stream.dataset": "rita", "tags": [], "processors": "- add_fields:\n target: event\n fields:\n category: network\n module: rita\n- if:\n log.file.path: beacons.csv\n then: \n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: rita.beacon\n- if:\n regexp:\n log.file.path: \"*connections.csv\"\n then: \n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: rita.connection\n- if:\n log.file.path: \"exploded-dns.csv\"\n then: \n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: rita.dns" }}}}}}' -#echo - -# Elasticsearch logs -echo -echo "Setting up Elasticsearch package policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "elasticsearch", "version": "1.0.0" }, "id": "elasticsearch-logs", "name": "elasticsearch-logs", "description": "Elasticsearch Logs", "namespace": "default", "inputs": { "elasticsearch-logfile": { "enabled": true, "streams": { "elasticsearch.audit": { "enabled": false, "vars": { "paths": [ "/var/log/elasticsearch/*_audit.json" ] } }, "elasticsearch.deprecation": { "enabled": false, "vars": { "paths": [ "/var/log/elasticsearch/*_deprecation.json" ] } }, "elasticsearch.gc": { "enabled": false, "vars": { "paths": [ "/var/log/elasticsearch/gc.log.[0-9]*", "/var/log/elasticsearch/gc.log" ] } }, "elasticsearch.server": { "enabled": true, "vars": { "paths": [ "/opt/so/log/elasticsearch/*.log" ] } }, "elasticsearch.slowlog": { "enabled": false, "vars": { "paths": [ "/var/log/elasticsearch/*_index_search_slowlog.json", "/var/log/elasticsearch/*_index_indexing_slowlog.json" ] } } } }, "elasticsearch-elasticsearch/metrics": { "enabled": false, "vars": { "hosts": [ "http://localhost:9200" ], "scope": "node" }, "streams": { "elasticsearch.stack_monitoring.ccr": { "enabled": false }, "elasticsearch.stack_monitoring.cluster_stats": { "enabled": false }, "elasticsearch.stack_monitoring.enrich": { "enabled": false }, "elasticsearch.stack_monitoring.index": { "enabled": false }, "elasticsearch.stack_monitoring.index_recovery": { "enabled": false, "vars": { "active.only": true } }, "elasticsearch.stack_monitoring.index_summary": { "enabled": false }, "elasticsearch.stack_monitoring.ml_job": { "enabled": false }, "elasticsearch.stack_monitoring.node": { "enabled": false }, "elasticsearch.stack_monitoring.node_stats": { "enabled": false }, "elasticsearch.stack_monitoring.pending_tasks": { "enabled": false }, "elasticsearch.stack_monitoring.shard": { "enabled": false } } } } }' -echo - -# Logstash logs -#echo -#echo "Setting up Logstash package policy..." -#curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "logstash", "version": "2.0.0" }, "id": "logstash-logs", "name": "logstash-logs", "description": "Logstash logs", "namespace": "default", "inputs": { "logstash-logfile": { "enabled": true, "streams": { "logstash.log": { "enabled": true, "vars": { "paths": [ "/opt/so/logs/logstash/logstash.log" ] } }, "logstash.slowlog": { "enabled": false, "vars": { "paths": [ "/var/log/logstash/logstash-slowlog-plain*.log", "/var/log/logstash/logstash-slowlog-json*.log" ] } } } }, "logstash-logstash/metrics": { "enabled": false, "vars": { "hosts": [ "http://localhost:9600" ], "period": "10s" }, "streams": { "logstash.stack_monitoring.node": { "enabled": false }, "logstash.stack_monitoring.node_stats": { "enabled": false } } } } }' -#echo - -# Kibana logs -#echo -#echo "Setting up Kibana package policy..." -#curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "kibana", "version": "2.0.0" }, "id": "kibana-logs", "name": "kibana-logs", "description": "Kibana logs", "namespace": "default", "inputs": { "kibana-logfile": { "enabled": true, "streams": { "kibana.audit": { "enabled": false, "vars": { "paths": [ "/opt/so/log/kibana/kibana.log" ] } }, "kibana.log": { "enabled": true, "vars": { "paths": [ "/opt/so/log/kibana/kibana.log" ] } } } }, "kibana-kibana/metrics": { "enabled": false, "vars": { "hosts": [ "http://localhost:5601" ] }, "streams": { "kibana.stack_monitoring.cluster_actions": { "enabled": false }, "kibana.stack_monitoring.cluster_rules": { "enabled": false }, "kibana.stack_monitoring.node_actions": { "enabled": false }, "kibana.stack_monitoring.node_rules": { "enabled": false }, "kibana.stack_monitoring.stats": { "enabled": false }, "kibana.stack_monitoring.status": { "enabled": false } } } } }' -#echo - -# Redis logs -echo -echo "Setting up Redis package policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "redis", "version": "1.4.0" }, "id": "redis-logs", "name": "redis-logs", "description": "Redis logs", "namespace": "default", "inputs": { "redis-logfile": { "enabled": true, "streams": { "redis.log": { "enabled": true, "vars": { "paths": [ "/opt/so/log/redis/redis.log" ], "tags": [ "redis-log" ], "preserve_original_event": false } } } }, "redis-redis": { "enabled": false, "streams": { "redis.slowlog": { "enabled": false, "vars": { "hosts": [ "127.0.0.1:6379" ], "password": "" } } } }, "redis-redis/metrics": { "enabled": false, "vars": { "hosts": [ "127.0.0.1:6379" ], "idle_timeout": "20s", "maxconn": 10, "network": "tcp", "password": "" }, "streams": { "redis.info": { "enabled": false, "vars": { "period": "10s" } }, "redis.key": { "enabled": false, "vars": { "key.patterns": "- limit: 20\n pattern: '*'\n", "period": "10s" } }, "redis.keyspace": { "enabled": false, "vars": { "period": "10s" } } } } } }' -echo - -# IDH logs -echo -echo "Setting up IDH package policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{"policy_id":"so-grid-nodes","package":{"name":"log","version":"1.1.1"},"id":"idh-logs","name":"idh-logs","namespace":"so","description":"IDH integration","inputs":{"logs-logfile":{"enabled":true,"streams":{"log.log":{"enabled":true,"vars":{"paths":["/nsm/idh/opencanary.log"],"data_stream.dataset":"idh","custom":"pipeline: common","processors": "\n- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n add_error_key: true\n- drop_fields:\n when:\n equals:\n logtype: \"1001\"\n fields: [\"src_host\", \"src_port\", \"dst_host\", \"dst_port\" ]\n ignore_missing: true\n- rename:\n fields:\n - from: \"src_host\"\n to: \"source.ip\"\n - from: \"src_port\"\n to: \"source.port\"\n - from: \"dst_host\"\n to: \"destination.host\"\n - from: \"dst_port\"\n to: \"destination.port\"\n ignore_missing: true\n- convert:\n fields:\n - {from: \"logtype\", to: \"event.code\", type: \"string\"}\n ignore_missing: true\n- drop_fields:\n fields: '\''[\"prospector\", \"input\", \"offset\", \"beat\"]'\''\n- add_fields:\n target: event\n fields:\n category: host\n module: opencanary","tags":[]}}}}}}' -echo - -# SOC - Server logs -echo -echo "Setting up SOC - Server Logs package policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{"package":{"name":"log","version":"1.1.2"},"name":"soc-server-logs","namespace":"so","description":"Security Onion Console Logs","policy_id":"so-grid-nodes","inputs":{"logs-logfile":{"enabled":true,"streams":{"log.log":{"enabled":true,"vars":{"paths":["/opt/so/log/soc/sensoroni-server.log"],"data_stream.dataset":"soc","custom":"pipeline: common","processors": "- decode_json_fields:\n fields: [\"message\"]\n target: \"soc\"\n process_array: true\n max_depth: 2\n add_error_key: true \n- add_fields:\n target: event\n fields:\n category: host\n module: soc\n dataset_temp: server\n- rename:\n fields:\n - from: \"soc.fields.sourceIp\"\n to: \"source.ip\"\n - from: \"soc.fields.status\"\n to: \"http.response.status_code\"\n - from: \"soc.fields.method\"\n to: \"http.request.method\"\n - from: \"soc.fields.path\"\n to: \"url.path\"\n - from: \"soc.message\"\n to: \"event.action\"\n - from: \"soc.level\"\n to: \"log.level\"\n ignore_missing: true","tags":[]}}}}}}' -echo - -# SOC - Sensoroni logs -echo -echo "Setting up SOC - Sensoroni Logs package policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{"package":{"name":"log","version":"1.1.2"},"name":"soc-sensoroni-logs","namespace":"so","description":"Security Onion - Sensoroni - Logs","policy_id":"so-grid-nodes","inputs":{"logs-logfile":{"enabled":true,"streams":{"log.log":{"enabled":true,"vars":{"paths":["/opt/so/log/sensoroni/sensoroni.log"],"data_stream.dataset":"soc","custom":"pipeline: common","processors": "- decode_json_fields:\n fields: [\"message\"]\n target: \"sensoroni\"\n process_array: true\n max_depth: 2\n add_error_key: true \n- add_fields:\n target: event\n fields:\n category: host\n module: soc\n dataset_temp: sensoroni\n- rename:\n fields:\n - from: \"sensoroni.fields.sourceIp\"\n to: \"source.ip\"\n - from: \"sensoroni.fields.status\"\n to: \"http.response.status_code\"\n - from: \"sensoroni.fields.method\"\n to: \"http.request.method\"\n - from: \"sensoroni.fields.path\"\n to: \"url.path\"\n - from: \"sensoroni.message\"\n to: \"event.action\"\n - from: \"sensoroni.level\"\n to: \"log.level\"\n ignore_missing: true","tags":[]}}}}}}' -echo - -# SOC - Elastic Auth Sync logs -echo -echo "Setting up SOC - Elastic Auth Sync Logs package policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{"package":{"name":"log","version":"1.1.2"},"name":"soc-auth-sync-logs","namespace":"so","description":"Security Onion - Elastic Auth Sync - Logs","policy_id":"so-grid-nodes","inputs":{"logs-logfile":{"enabled":true,"streams":{"log.log":{"enabled":true,"vars":{"paths":["/opt/so/log/soc/sync.log"],"data_stream.dataset":"soc","custom":"pipeline: common","processors": "- dissect:\n tokenizer: \"%{event.action}\"\n field: \"message\"\n target_prefix: \"\"\n- add_fields:\n target: event\n fields:\n category: host\n module: soc\n dataset_temp: auth_sync","tags":[]}}}}}}' -echo - -# SOC - Salt Relay logs -echo -echo "Setting up SOC - Salt_Relay Logs package policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{"package":{"name":"log","version":"1.1.2"},"name":"soc-salt-relay-logs","namespace":"so","description":"Security Onion - Salt Relay - Logs","policy_id":"so-grid-nodes","inputs":{"logs-logfile":{"enabled":true,"streams":{"log.log":{"enabled":true,"vars":{"paths":["/opt/so/log/soc/salt-relay.log"],"data_stream.dataset":"soc","custom":"pipeline: common","processors": "- dissect:\n tokenizer: \"%{soc.ts} | %{event.action}\"\n field: \"message\"\n target_prefix: \"\"\n- add_fields:\n target: event\n fields:\n category: host\n module: soc\n dataset_temp: salt_relay","tags":[]}}}}}}' -echo +# Grid Nodes +for INTEGRATION in /opt/so/saltstack/default/salt/elasticfleet/files/integrations/grid-nodes/*.json +do + printf "\n\nGrid Nodes Policy - Loading $INTEGRATION\n" + elastic_fleet_integration_create "@$INTEGRATION" +done \ No newline at end of file diff --git a/salt/common/tools/sbin/so-elastic-fleet-setup b/salt/common/tools/sbin/so-elastic-fleet-setup index 86b9c1107..7fe336625 100755 --- a/salt/common/tools/sbin/so-elastic-fleet-setup +++ b/salt/common/tools/sbin/so-elastic-fleet-setup @@ -8,40 +8,6 @@ . /usr/sbin/so-common -elastic_fleet_policy_create() { - - NAME=$1 - DESC=$2 - FLEETSERVER=$3 - - JSON_STRING=$( jq -n \ - --arg NAME "$NAME" \ - --arg DESC "$DESC" \ - --arg FLEETSERVER "$FLEETSERVER" \ - '{"name": $NAME,"id":$NAME,"description":$DESC,"namespace":"default","monitoring_enabled":["logs"],"inactivity_timeout":1209600,"has_fleet_server":$FLEETSERVER}' - ) - # Create Fleet Policy - curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/agent_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" - -} - -elastic_fleet_policy_update() { - - POLICYID=$1 - JSON_STRING_UPDATE=$2 - - curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/agent_policies/$POLICYID" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING_UPDATE" -} - -elastic_fleet_integration_create() { - - JSONBLOB=$1 - - #curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/agent_policies/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSONBLOB" - curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSONBLOB" - -} - printf "\n### Create ES Token ###\n" ESTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/service_tokens" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' | jq -r .value) @@ -79,30 +45,22 @@ printf "\n\n" # Manager Fleet Server Host elastic_fleet_policy_create "FleetServer_{{ GLOBALS.hostname }}" "Fleet Server - {{ GLOBALS.hostname }}" "true" | jq -#elastic_fleet_policy_update "FleetServer_{{ GLOBALS.hostname }}" "@/opt/so/saltstack/default/salt/elasticfleet/files/FleeServerHost_Fixup.json" -#Temp Fixup +#Temp Fixup for ES Output bug JSON_STRING=$( jq -n \ --arg NAME "FleetServer_{{ GLOBALS.hostname }}" \ '{"name": $NAME,"description": $NAME,"namespace":"default","monitoring_enabled":["logs"],"inactivity_timeout":120,"data_output_id":"so-manager_elasticsearch"}' ) curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/agent_policies/FleetServer_{{ GLOBALS.hostname }}" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" +# Initial Endpoints Policy +elastic_fleet_policy_create "endpoints-initial" "Initial Endpoint Policy" "false" -# Initial Endpoints -elastic_fleet_policy_create "endpoints-initial" "Initial Endpoint Policy" "false" | jq -for INTEGRATION in /opt/so/saltstack/default/salt/elasticfleet/files/integrations/endpoints-initial/*.json -do - elastic_fleet_integration_create "@$INTEGRATION" | jq -done - -# Grid Nodes +# Grid Nodes Policy elastic_fleet_policy_create "so-grid-nodes" "SO Grid Node Policy" "false" -for INTEGRATION in /opt/so/saltstack/default/salt/elasticfleet/files/integrations/grid-nodes/*.json -do - elastic_fleet_integration_create "@$INTEGRATION" | jq -done +# Load Integrations for default policies +so-elastic-fleet-integration-policy-load ### Finalization ### diff --git a/salt/common/tools/sbin/so-minion b/salt/common/tools/sbin/so-minion index cf50a49d7..2f506863d 100755 --- a/salt/common/tools/sbin/so-minion +++ b/salt/common/tools/sbin/so-minion @@ -237,7 +237,23 @@ function create_fleet_policy() { # Update Fleet Policy - ES Output curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/agent_policies/FleetServer_$LSHOSTNAME" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING_UPDATE" -} +} + +function update_fleet_host_urls() { + # Query for current Fleet Host URLs & append New Fleet Node Hostname & IP + JSON_STRING=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/fleet_server_hosts' | jq --arg HOSTNAME "https://$LSHOSTNAME:8220" --arg IP "https://$MAINIP:8220" -r '.items[].host_urls += [ $HOSTNAME, $IP ] | {"name":"Default","host_urls": .items[].host_urls,"is_default":true,"proxy_id":null}') + + # Update Fleet Host URLs + curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/fleet_server_hosts/fleet-default-fleet-server-host" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" +} + +function update_logstash_outputs() { + # Query for current Logstash outputs & append New Fleet Node Hostname & IP + JSON_STRING=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/outputs/so-manager_logstash' | jq --arg HOSTNAME "$LSHOSTNAME:5055" --arg IP "$MAINIP:5055" -r '.item.hosts += [ $HOSTNAME, $IP ] | {"name":"grid-logstash","type":"logstash","hosts": .item.hosts,"is_default":true,"is_default_monitoring":true,"config_yaml":""}') + + # Update Logstash Outputs + curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/outputs/so-manager_logstash" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" +} function updateMine() { salt "$MINION_ID" mine.send network.ip_addrs interface="$MNIC" @@ -255,6 +271,8 @@ function createFLEET() { add_fleet_to_minion add_logstash_to_minion create_fleet_policy + update_fleet_host_urls + update_logstash_outputs } function createIDH() { diff --git a/salt/elasticfleet/files/integrations/endpoints-initial/system-endpoints.json b/salt/elasticfleet/files/integrations/endpoints-initial/system-endpoints.json new file mode 100644 index 000000000..eb5ef74e9 --- /dev/null +++ b/salt/elasticfleet/files/integrations/endpoints-initial/system-endpoints.json @@ -0,0 +1,76 @@ +{ + "package": { + "name": "system", + "version": "1.25.2" + }, + "name": "system-endpoints", + "namespace": "default", + "policy_id": "endpoints-initial", + "inputs": { + "system-logfile": { + "enabled": true, + "streams": { + "system.auth": { + "enabled": true, + "vars": { + "ignore_older": "72h", + "paths": [ + "/var/log/auth.log*", + "/var/log/secure*" + ], + "preserve_original_event": false, + "tags": [ + "system-auth" + ] + } + }, + "system.syslog": { + "enabled": true, + "vars": { + "paths": [ + "/var/log/messages*", + "/var/log/syslog*" + ], + "tags": [], + "ignore_older": "72h" + } + } + } + }, + "system-winlog": { + "enabled": true, + "streams": { + "system.application": { + "enabled": true, + "vars": { + "preserve_original_event": false, + "ignore_older": "72h", + "language": 0, + "tags": [] + } + }, + "system.security": { + "enabled": true, + "vars": { + "preserve_original_event": false, + "ignore_older": "72h", + "language": 0, + "tags": [] + } + }, + "system.system": { + "enabled": true, + "vars": { + "preserve_original_event": false, + "ignore_older": "72h", + "language": 0, + "tags": [] + } + } + } + }, + "system-system/metrics": { + "enabled": false + } + } +} diff --git a/salt/elasticfleet/files/integrations/endpoints-initial/windows-endpoints.json b/salt/elasticfleet/files/integrations/endpoints-initial/windows-endpoints.json new file mode 100644 index 000000000..9b647c8ab --- /dev/null +++ b/salt/elasticfleet/files/integrations/endpoints-initial/windows-endpoints.json @@ -0,0 +1,59 @@ +{ + "package": { + "name": "windows", + "version": "1.19.1" + }, + "name": "windows-endpoints", + "namespace": "default", + "policy_id": "endpoints-initial", + "inputs": { + "windows-winlog": { + "enabled": true, + "streams": { + "windows.forwarded": { + "enabled": true, + "vars": { + "preserve_original_event": false, + "ignore_older": "72h", + "language": 0, + "tags": [ + "forwarded" + ] + } + }, + "windows.powershell": { + "enabled": true, + "vars": { + "preserve_original_event": false, + "event_id": "400, 403, 600, 800", + "ignore_older": "72h", + "language": 0, + "tags": [] + } + }, + "windows.powershell_operational": { + "enabled": true, + "vars": { + "preserve_original_event": false, + "event_id": "4103, 4104, 4105, 4106", + "ignore_older": "72h", + "language": 0, + "tags": [] + } + }, + "windows.sysmon_operational": { + "enabled": true, + "vars": { + "preserve_original_event": false, + "ignore_older": "72h", + "language": 0, + "tags": [] + } + } + } + }, + "windows-windows/metrics": { + "enabled": false + } + } +} diff --git a/salt/elasticfleet/files/integrations/grid-nodes/elasticsearch-logs.json b/salt/elasticfleet/files/integrations/grid-nodes/elasticsearch-logs.json new file mode 100644 index 000000000..1a93a9022 --- /dev/null +++ b/salt/elasticfleet/files/integrations/grid-nodes/elasticsearch-logs.json @@ -0,0 +1,106 @@ +{ + "package": { + "name": "elasticsearch", + "version": "1.0.0" + }, + "name": "elasticsearch-logs", + "namespace": "default", + "description": "Elasticsearch Logs", + "policy_id": "so-grid-nodes", + "inputs": { + "elasticsearch-logfile": { + "enabled": true, + "streams": { + "elasticsearch.audit": { + "enabled": false, + "vars": { + "paths": [ + "/var/log/elasticsearch/*_audit.json" + ] + } + }, + "elasticsearch.deprecation": { + "enabled": false, + "vars": { + "paths": [ + "/var/log/elasticsearch/*_deprecation.json" + ] + } + }, + "elasticsearch.gc": { + "enabled": false, + "vars": { + "paths": [ + "/var/log/elasticsearch/gc.log.[0-9]*", + "/var/log/elasticsearch/gc.log" + ] + } + }, + "elasticsearch.server": { + "enabled": true, + "vars": { + "paths": [ + "/opt/so/log/elasticsearch/*.log" + ] + } + }, + "elasticsearch.slowlog": { + "enabled": false, + "vars": { + "paths": [ + "/var/log/elasticsearch/*_index_search_slowlog.json", + "/var/log/elasticsearch/*_index_indexing_slowlog.json" + ] + } + } + } + }, + "elasticsearch-elasticsearch/metrics": { + "enabled": false, + "vars": { + "hosts": [ + "http://localhost:9200" + ], + "scope": "node" + }, + "streams": { + "elasticsearch.stack_monitoring.ccr": { + "enabled": false + }, + "elasticsearch.stack_monitoring.cluster_stats": { + "enabled": false + }, + "elasticsearch.stack_monitoring.enrich": { + "enabled": false + }, + "elasticsearch.stack_monitoring.index": { + "enabled": false + }, + "elasticsearch.stack_monitoring.index_recovery": { + "enabled": false, + "vars": { + "active.only": true + } + }, + "elasticsearch.stack_monitoring.index_summary": { + "enabled": false + }, + "elasticsearch.stack_monitoring.ml_job": { + "enabled": false + }, + "elasticsearch.stack_monitoring.node": { + "enabled": false + }, + "elasticsearch.stack_monitoring.node_stats": { + "enabled": false + }, + "elasticsearch.stack_monitoring.pending_tasks": { + "enabled": false + }, + "elasticsearch.stack_monitoring.shard": { + "enabled": false + } + } + } + } +} diff --git a/salt/elasticfleet/files/integrations/grid-nodes/import-evtx-logs.json b/salt/elasticfleet/files/integrations/grid-nodes/import-evtx-logs.json new file mode 100644 index 000000000..aa54f7226 --- /dev/null +++ b/salt/elasticfleet/files/integrations/grid-nodes/import-evtx-logs.json @@ -0,0 +1,29 @@ +{ + "package": { + "name": "log", + "version": "1.1.0" + }, + "name": "import-evtx-logs", + "namespace": "so", + "description": "Import Windows EVTX logs", + "policy_id": "so-grid-nodes", + "inputs": { + "logs-logfile": { + "enabled": true, + "streams": { + "log.log": { + "enabled": true, + "vars": { + "paths": [ + "/nsm/import/*/evtx/data.json" + ], + "data_stream.dataset": "import", + "tags": [], + "processors": "- dissect:\n tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"\n- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n- add_fields:\n target: event\n fields:\n module: windows_eventlog\n imported: true", + "custom": "pipeline: import.wel" + } + } + } + } + } +} diff --git a/salt/elasticfleet/files/integrations/grid-nodes/import-suricata-logs.json b/salt/elasticfleet/files/integrations/grid-nodes/import-suricata-logs.json new file mode 100644 index 000000000..e8b05a7d1 --- /dev/null +++ b/salt/elasticfleet/files/integrations/grid-nodes/import-suricata-logs.json @@ -0,0 +1,29 @@ +{ + "package": { + "name": "log", + "version": "1.1.0" + }, + "name": "import-suricata-logs", + "namespace": "so", + "description": "Import Suricata logs", + "policy_id": "so-grid-nodes", + "inputs": { + "logs-logfile": { + "enabled": true, + "streams": { + "log.log": { + "enabled": true, + "vars": { + "paths": [ + "/nsm/import/*/suricata/eve*.json" + ], + "data_stream.dataset": "import", + "tags": [], + "processors": "- add_fields:\n target: event\n fields:\n category: network\n module: suricata\n imported: true\n- dissect:\n tokenizer: \"/nsm/import/%{import.id}/suricata/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"", + "custom": "pipeline: suricata.common" + } + } + } + } + } +} diff --git a/salt/elasticfleet/files/integrations/grid-nodes/import-zeek-logs.json b/salt/elasticfleet/files/integrations/grid-nodes/import-zeek-logs.json new file mode 100644 index 000000000..ab752eaec --- /dev/null +++ b/salt/elasticfleet/files/integrations/grid-nodes/import-zeek-logs.json @@ -0,0 +1,29 @@ +{ + "package": { + "name": "log", + "version": "1.1.0" + }, + "name": "import-zeek-logs", + "namespace": "so", + "description": "Zeek Import logs", + "policy_id": "so-grid-nodes", + "inputs": { + "logs-logfile": { + "enabled": true, + "streams": { + "log.log": { + "enabled": true, + "vars": { + "paths": [ + "/nsm/import/*/zeek/logs/*.log" + ], + "data_stream.dataset": "import", + "tags": [], + "processors": "- dissect:\n tokenizer: \"/nsm/import/%{import.id}/zeek/logs/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"\n- script:\n lang: javascript\n source: >\n function process(event) {\n var pl = event.Get(\"import.file\").slice(0,-4);\n event.Put(\"@metadata.pipeline\", \"zeek.\" + pl);\n }\n- add_fields:\n target: event\n fields:\n category: network\n module: zeek\n imported: true\n- add_tags:\n tags: \"ics\"\n when:\n regexp:\n import.file: \"^bacnet*|^bsap*|^cip*|^cotp*|^dnp3*|^ecat*|^enip*|^modbus*|^opcua*|^profinet*|^s7comm*\"", + "custom": "exclude_files: [\"broker|capture_loss|ecat_arp_info|loaded_scripts|packet_filter|stats|stderr|stdout.log$\"]\n" + } + } + } + } + } +} diff --git a/salt/elasticfleet/files/integrations/grid-nodes/osquery-grid-nodes.json b/salt/elasticfleet/files/integrations/grid-nodes/osquery-grid-nodes.json new file mode 100644 index 000000000..92f10f591 --- /dev/null +++ b/salt/elasticfleet/files/integrations/grid-nodes/osquery-grid-nodes.json @@ -0,0 +1,20 @@ +{ + "package": { + "name": "osquery_manager", + "version": "1.6.0" + }, + "name": "osquery-grid-nodes", + "namespace": "default", + "policy_id": "so-grid-nodes", + "inputs": { + "osquery_manager-osquery": { + "enabled": true, + "streams": { + "osquery_manager.result": { + "enabled": true, + "vars": {} + } + } + } + } +} diff --git a/salt/elasticfleet/files/integrations/grid-nodes/redis-logs.json b/salt/elasticfleet/files/integrations/grid-nodes/redis-logs.json new file mode 100644 index 000000000..8b71cbac7 --- /dev/null +++ b/salt/elasticfleet/files/integrations/grid-nodes/redis-logs.json @@ -0,0 +1,76 @@ +{ + "package": { + "name": "redis", + "version": "1.4.0" + }, + "name": "redis-logs", + "namespace": "default", + "description": "Redis logs", + "policy_id": "so-grid-nodes", + "inputs": { + "redis-logfile": { + "enabled": true, + "streams": { + "redis.log": { + "enabled": true, + "vars": { + "paths": [ + "/opt/so/log/redis/redis.log" + ], + "tags": [ + "redis-log" + ], + "preserve_original_event": false + } + } + } + }, + "redis-redis": { + "enabled": false, + "streams": { + "redis.slowlog": { + "enabled": false, + "vars": { + "hosts": [ + "127.0.0.1:6379" + ], + "password": "" + } + } + } + }, + "redis-redis/metrics": { + "enabled": false, + "vars": { + "hosts": [ + "127.0.0.1:6379" + ], + "idle_timeout": "20s", + "maxconn": 10, + "network": "tcp", + "password": "" + }, + "streams": { + "redis.info": { + "enabled": false, + "vars": { + "period": "10s" + } + }, + "redis.key": { + "enabled": false, + "vars": { + "key.patterns": "- limit: 20\n pattern: *\n", + "period": "10s" + } + }, + "redis.keyspace": { + "enabled": false, + "vars": { + "period": "10s" + } + } + } + } + } +} diff --git a/salt/elasticfleet/files/integrations/grid-nodes/soc-auth-sync-logs.json b/salt/elasticfleet/files/integrations/grid-nodes/soc-auth-sync-logs.json new file mode 100644 index 000000000..e5067490d --- /dev/null +++ b/salt/elasticfleet/files/integrations/grid-nodes/soc-auth-sync-logs.json @@ -0,0 +1,29 @@ +{ + "package": { + "name": "log", + "version": "1.1.2" + }, + "name": "soc-auth-sync-logs", + "namespace": "so", + "description": "Security Onion - Elastic Auth Sync - Logs", + "policy_id": "so-grid-nodes", + "inputs": { + "logs-logfile": { + "enabled": true, + "streams": { + "log.log": { + "enabled": true, + "vars": { + "paths": [ + "/opt/so/log/soc/sync.log" + ], + "data_stream.dataset": "soc", + "tags": [], + "processors": "- dissect:\n tokenizer: \"%{event.action}\"\n field: \"message\"\n target_prefix: \"\"\n- add_fields:\n target: event\n fields:\n category: host\n module: soc\n dataset_temp: auth_sync", + "custom": "pipeline: common" + } + } + } + } + } +} diff --git a/salt/elasticfleet/files/integrations/grid-nodes/soc-salt-relay-logs.json b/salt/elasticfleet/files/integrations/grid-nodes/soc-salt-relay-logs.json new file mode 100644 index 000000000..90ac79824 --- /dev/null +++ b/salt/elasticfleet/files/integrations/grid-nodes/soc-salt-relay-logs.json @@ -0,0 +1,29 @@ +{ + "package": { + "name": "log", + "version": "1.1.2" + }, + "name": "soc-salt-relay-logs", + "namespace": "so", + "description": "Security Onion - Salt Relay - Logs", + "policy_id": "so-grid-nodes", + "inputs": { + "logs-logfile": { + "enabled": true, + "streams": { + "log.log": { + "enabled": true, + "vars": { + "paths": [ + "/opt/so/log/soc/salt-relay.log" + ], + "data_stream.dataset": "soc", + "tags": [], + "processors": "- dissect:\n tokenizer: \"%{soc.ts} | %{event.action}\"\n field: \"message\"\n target_prefix: \"\"\n- add_fields:\n target: event\n fields:\n category: host\n module: soc\n dataset_temp: salt_relay", + "custom": "pipeline: common" + } + } + } + } + } +} diff --git a/salt/elasticfleet/files/integrations/grid-nodes/soc-sensoroni-logs.json b/salt/elasticfleet/files/integrations/grid-nodes/soc-sensoroni-logs.json new file mode 100644 index 000000000..8d94f2d8d --- /dev/null +++ b/salt/elasticfleet/files/integrations/grid-nodes/soc-sensoroni-logs.json @@ -0,0 +1,29 @@ +{ + "package": { + "name": "log", + "version": "1.1.2" + }, + "name": "soc-sensoroni-logs", + "namespace": "so", + "description": "Security Onion - Sensoroni - Logs", + "policy_id": "so-grid-nodes", + "inputs": { + "logs-logfile": { + "enabled": true, + "streams": { + "log.log": { + "enabled": true, + "vars": { + "paths": [ + "/opt/so/log/sensoroni/sensoroni.log" + ], + "data_stream.dataset": "soc", + "tags": [], + "processors": "- decode_json_fields:\n fields: [\"message\"]\n target: \"sensoroni\"\n process_array: true\n max_depth: 2\n add_error_key: true \n- add_fields:\n target: event\n fields:\n category: host\n module: soc\n dataset_temp: sensoroni\n- rename:\n fields:\n - from: \"sensoroni.fields.sourceIp\"\n to: \"source.ip\"\n - from: \"sensoroni.fields.status\"\n to: \"http.response.status_code\"\n - from: \"sensoroni.fields.method\"\n to: \"http.request.method\"\n - from: \"sensoroni.fields.path\"\n to: \"url.path\"\n - from: \"sensoroni.message\"\n to: \"event.action\"\n - from: \"sensoroni.level\"\n to: \"log.level\"\n ignore_missing: true", + "custom": "pipeline: common" + } + } + } + } + } +} diff --git a/salt/elasticfleet/files/integrations/grid-nodes/soc-server-logs.json b/salt/elasticfleet/files/integrations/grid-nodes/soc-server-logs.json new file mode 100644 index 000000000..908b1a782 --- /dev/null +++ b/salt/elasticfleet/files/integrations/grid-nodes/soc-server-logs.json @@ -0,0 +1,29 @@ +{ + "package": { + "name": "log", + "version": "1.1.2" + }, + "name": "soc-server-logs", + "namespace": "so", + "description": "Security Onion Console Logs", + "policy_id": "so-grid-nodes", + "inputs": { + "logs-logfile": { + "enabled": true, + "streams": { + "log.log": { + "enabled": true, + "vars": { + "paths": [ + "/opt/so/log/soc/sensoroni-server.log" + ], + "data_stream.dataset": "soc", + "tags": [], + "processors": "- decode_json_fields:\n fields: [\"message\"]\n target: \"soc\"\n process_array: true\n max_depth: 2\n add_error_key: true \n- add_fields:\n target: event\n fields:\n category: host\n module: soc\n dataset_temp: server\n- rename:\n fields:\n - from: \"soc.fields.sourceIp\"\n to: \"source.ip\"\n - from: \"soc.fields.status\"\n to: \"http.response.status_code\"\n - from: \"soc.fields.method\"\n to: \"http.request.method\"\n - from: \"soc.fields.path\"\n to: \"url.path\"\n - from: \"soc.message\"\n to: \"event.action\"\n - from: \"soc.level\"\n to: \"log.level\"\n ignore_missing: true", + "custom": "pipeline: common" + } + } + } + } + } +} diff --git a/salt/elasticfleet/files/integrations/grid-nodes/strelka-logs.json b/salt/elasticfleet/files/integrations/grid-nodes/strelka-logs.json new file mode 100644 index 000000000..75bd9d73c --- /dev/null +++ b/salt/elasticfleet/files/integrations/grid-nodes/strelka-logs.json @@ -0,0 +1,29 @@ +{ + "package": { + "name": "log", + "version": "1.1.0" + }, + "name": "strelka-logs", + "namespace": "so", + "description": "Strelka logs", + "policy_id": "so-grid-nodes", + "inputs": { + "logs-logfile": { + "enabled": true, + "streams": { + "log.log": { + "enabled": true, + "vars": { + "paths": [ + "/nsm/strelka/log/strelka.log" + ], + "data_stream.dataset": "strelka", + "tags": [], + "processors": "- add_fields:\n target: event\n fields:\n category: file\n module: strelka", + "custom": "pipeline: strelka.file" + } + } + } + } + } +} diff --git a/salt/elasticfleet/files/integrations/grid-nodes/suricata-logs.json b/salt/elasticfleet/files/integrations/grid-nodes/suricata-logs.json new file mode 100644 index 000000000..623513c34 --- /dev/null +++ b/salt/elasticfleet/files/integrations/grid-nodes/suricata-logs.json @@ -0,0 +1,29 @@ +{ + "package": { + "name": "log", + "version": "1.1.0" + }, + "name": "suricata-logs", + "namespace": "so", + "description": "Suricata integration", + "policy_id": "so-grid-nodes", + "inputs": { + "logs-logfile": { + "enabled": true, + "streams": { + "log.log": { + "enabled": true, + "vars": { + "paths": [ + "/nsm/suricata/eve*.json" + ], + "data_stream.dataset": "suricata", + "tags": [], + "processors": "- add_fields:\n target: event\n fields:\n category: network\n module: suricata", + "custom": "pipeline: suricata.common" + } + } + } + } + } +} diff --git a/salt/elasticfleet/files/integrations/grid-nodes/syslog-tcp-514.json b/salt/elasticfleet/files/integrations/grid-nodes/syslog-tcp-514.json new file mode 100644 index 000000000..a6321f19b --- /dev/null +++ b/salt/elasticfleet/files/integrations/grid-nodes/syslog-tcp-514.json @@ -0,0 +1,32 @@ +{ + "package": { + "name": "tcp", + "version": "1.5.0" + }, + "name": "syslog-tcp-514", + "namespace": "so", + "description": "Syslog Over TCP Port 514", + "policy_id": "so-grid-nodes", + "inputs": { + "tcp-tcp": { + "enabled": true, + "streams": { + "tcp.generic": { + "enabled": true, + "vars": { + "listen_address": "0.0.0.0", + "listen_port": "514", + "data_stream.dataset": "syslog", + "pipeline": "syslog", + "processors": "- add_fields:\n target: event\n fields:\n module: syslog", + "tags": [ + "syslog" + ], + "syslog_options": "field: message\n#format: auto\n#timezone: Local", + "ssl": "" + } + } + } + } + } +} diff --git a/salt/elasticfleet/files/integrations/grid-nodes/syslog-udp-514.json b/salt/elasticfleet/files/integrations/grid-nodes/syslog-udp-514.json new file mode 100644 index 000000000..b9dd14640 --- /dev/null +++ b/salt/elasticfleet/files/integrations/grid-nodes/syslog-udp-514.json @@ -0,0 +1,33 @@ +{ + "package": { + "name": "udp", + "version": "1.5.0" + }, + "name": "syslog-udp-514", + "namespace": "so", + "description": "Syslog over UDP Port 514", + "policy_id": "so-grid-nodes", + "inputs": { + "udp-udp": { + "enabled": true, + "streams": { + "udp.generic": { + "enabled": true, + "vars": { + "listen_address": "0.0.0.0", + "listen_port": "514", + "data_stream.dataset": "syslog", + "pipeline": "syslog", + "max_message_size": "10KiB", + "keep_null": false, + "processors": "- add_fields:\n target: event\n fields: \n module: syslog\n", + "tags": [ + "syslog" + ], + "syslog_options": "field: message\n#format: auto\n#timezone: Local" + } + } + } + } + } +} diff --git a/salt/elasticfleet/files/integrations/grid-nodes/system-grid-nodes.json b/salt/elasticfleet/files/integrations/grid-nodes/system-grid-nodes.json new file mode 100644 index 000000000..866bae8cd --- /dev/null +++ b/salt/elasticfleet/files/integrations/grid-nodes/system-grid-nodes.json @@ -0,0 +1,47 @@ +{ + "package": { + "name": "system", + "version": "1.25.2" + }, + "name": "system-grid-nodes", + "namespace": "default", + "policy_id": "so-grid-nodes", + "inputs": { + "system-logfile": { + "enabled": true, + "streams": { + "system.auth": { + "enabled": true, + "vars": { + "ignore_older": "72h", + "paths": [ + "/var/log/auth.log*", + "/var/log/secure*" + ], + "preserve_original_event": false, + "tags": [ + "system-auth" + ] + } + }, + "system.syslog": { + "enabled": true, + "vars": { + "paths": [ + "/var/log/messages*", + "/var/log/syslog*" + ], + "tags": [], + "ignore_older": "72h" + } + } + } + }, + "system-winlog": { + "enabled": false + }, + "system-system/metrics": { + "enabled": false + } + } +} diff --git a/salt/elasticfleet/files/integrations/grid-nodes/zeek-logs.json b/salt/elasticfleet/files/integrations/grid-nodes/zeek-logs.json new file mode 100644 index 000000000..a4e0c94ee --- /dev/null +++ b/salt/elasticfleet/files/integrations/grid-nodes/zeek-logs.json @@ -0,0 +1,29 @@ +{ + "package": { + "name": "log", + "version": "1.1.0" + }, + "name": "zeek-logs", + "namespace": "so", + "description": "Zeek logs", + "policy_id": "so-grid-nodes", + "inputs": { + "logs-logfile": { + "enabled": true, + "streams": { + "log.log": { + "enabled": true, + "vars": { + "paths": [ + "/nsm/zeek/logs/current/*.log" + ], + "data_stream.dataset": "zeek", + "tags": [], + "processors": "- dissect:\n tokenizer: \"/nsm/zeek/logs/current/%{pipeline}.log\"\n field: \"log.file.path\"\n trim_chars: \".log\"\n target_prefix: \"\"\n- script:\n lang: javascript\n source: >\n function process(event) {\n var pl = event.Get(\"pipeline\");\n event.Put(\"@metadata.pipeline\", \"zeek.\" + pl);\n }\n- add_fields:\n target: event\n fields:\n category: network\n module: zeek\n- add_tags:\n tags: \"ics\"\n when:\n regexp:\n pipeline: \"^bacnet*|^bsap*|^cip*|^cotp*|^dnp3*|^ecat*|^enip*|^modbus*|^opcua*|^profinet*|^s7comm*\"", + "custom": "exclude_files: [\"broker|capture_loss|ecat_arp_info|loaded_scripts|packet_filter|stats|stderr|stdout.log$\"]\n" + } + } + } + } + } +} diff --git a/salt/top.sls b/salt/top.sls index 8cad2816c..372c64115 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -347,6 +347,7 @@ base: - firewall - logstash - elasticfleet + - elasticfleet.install_agent_grid - schedule - docker_clean diff --git a/setup/so-setup b/setup/so-setup index ddf39fdee..3259fd079 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -634,6 +634,8 @@ if ! [[ -f $install_opt_file ]]; then logCmd "salt-call state.apply -l info manager" logCmd "salt-call state.apply influxdb -l info" logCmd "salt-call state.highstate -l info" + title "Setting up Kibana Default Space" + logCmd "so-kibana-space-defaults" add_web_user info "Restarting SOC to pick up initial user" logCmd "so-soc-restart"