Add integrations and cleanup

salt/common/tools/sbin/so-common

@@ -160,6 +160,39 @@ disable_fastestmirror() {
 	sed -i 's/enabled=1/enabled=0/' /etc/yum/pluginconf.d/fastestmirror.conf
 }
 
+elastic_fleet_integration_create() {
+
+    JSON_STRING=$1
+
+    curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
+}
+
+elastic_fleet_policy_create() {
+
+    NAME=$1
+    DESC=$2
+    FLEETSERVER=$3
+
+    JSON_STRING=$( jq -n \
+                    --arg NAME "$NAME" \
+                    --arg DESC "$DESC" \
+                    --arg FLEETSERVER "$FLEETSERVER" \
+                    '{"name": $NAME,"id":$NAME,"description":$DESC,"namespace":"default","monitoring_enabled":["logs"],"inactivity_timeout":1209600,"has_fleet_server":$FLEETSERVER}'
+                    )
+    # Create Fleet Policy
+    curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/agent_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" 
+
+}
+
+elastic_fleet_policy_update() {
+
+    POLICYID=$1
+    JSON_STRING=$2
+
+    curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/agent_policies/$POLICYID" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
+}
+
+
 elastic_license() {
 
 read -r -d '' message <<- EOM

salt/common/tools/sbin/so-elastic-fleet-integration-policy-load

@@ -6,132 +6,16 @@
 
 . /usr/sbin/so-common
 
-{%- set ZEEKVER = salt['pillar.get']('global:mdengine', '') %}
+# Initial Endpoints
-{%- set STRELKAENABLED = salt['pillar.get']('strelka:enabled', '0') %}
+for INTEGRATION in /opt/so/saltstack/default/salt/elasticfleet/files/integrations/endpoints-initial/*.json
-{%- set RITAENABLED = salt['pillar.get']('rita:enabled', False) %}
+do
+  printf "\n\nInitial Endpoint Policy - Loading $INTEGRATION\n"
+  elastic_fleet_integration_create "@$INTEGRATION"
+done
 
-wait_for_web_response "http://localhost:5601/api/spaces/space/default" "default" 300 "curl -K /opt/so/conf/elasticsearch/curl.config"
+# Grid Nodes
-
+for INTEGRATION in /opt/so/saltstack/default/salt/elasticfleet/files/integrations/grid-nodes/*.json
-# Let's snag a cookie from Kibana
+do
-SESSIONCOOKIE=$(curl -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}')
+  printf "\n\nGrid Nodes Policy - Loading $INTEGRATION\n"
-
+  elastic_fleet_integration_create "@$INTEGRATION"
-# Disable certain Features from showing up in the Kibana UI
+done
-echo
-echo "Disable certain Features from showing up in the Kibana UI"
-so-kibana-space-defaults
-echo
-
-# Suricata logs
-echo
-echo "Setting up Suricata package policy..."
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "suricata-logs", "name": "suricata-logs", "description": "Suricata integration", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/suricata/eve*.json" ], "data_stream.dataset": "suricata", "tags": [],"processors": "- add_fields:\n    target: event\n    fields:\n      category: network\n      module: suricata", "custom": "pipeline: suricata.common" }}}}}}'
-echo
-
-# Zeek logs
-echo
-echo "Setting up Zeek package policy..."
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "name": "zeek-logs", "description": "Zeek logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": ["/nsm/zeek/logs/current/*.log"], "data_stream.dataset": "zeek", "tags": [], "processors": "- dissect:\n    tokenizer: \"/nsm/zeek/logs/current/%{pipeline}.log\"\n    field: \"log.file.path\"\n    trim_chars: \".log\"\n    target_prefix: \"\"\n- script:\n      lang: javascript\n      source: >\n        function process(event) {\n          var pl = event.Get(\"pipeline\");\n          event.Put(\"@metadata.pipeline\", \"zeek.\" + pl);\n        }\n- add_fields:\n    target: event\n    fields:\n      category: network\n      module: zeek\n- add_tags:\n    tags: \"ics\"\n    when:\n      regexp:\n        pipeline: \"^bacnet*|^bsap*|^cip*|^cotp*|^dnp3*|^ecat*|^enip*|^modbus*|^opcua*|^profinet*|^s7comm*\"", "custom": "exclude_files: [\"broker|capture_loss|ecat_arp_info|loaded_scripts|packet_filter|stats|stderr|stdout.log$\"]\n" } } } } } }'
-echo
-
-
-# Import - EVTX
-echo
-echo "Setting up EVTX import package policy..."
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{ "package": { "name": "log", "version": "1.1.0" }, "name": "import-evtx-logs", "namespace": "so", "description": "Import Windows EVTX logs", "policy_id": "so-grid-nodes", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/import/*/evtx/data.json" ], "data_stream.dataset": "import", "custom": "pipeline: import.wel", "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- add_fields:\n    target: event\n    fields:\n      module: windows_eventlog\n      imported: true", "tags": [] } } } } } }'
-echo
-
-# Import - Suricata logs
-echo
-echo "Setting up Suricata import package policy..."
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "import-suricata-logs", "name": "import-suricata-logs", "description": "Import Suricata logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": ["/nsm/import/*/suricata/eve*.json"], "data_stream.dataset": "import", "tags": [], "processors": "- add_fields:\n    target: event\n    fields:\n      category: network\n      module: suricata\n      imported: true\n- dissect:\n      tokenizer: \"/nsm/import/%{import.id}/suricata/%{import.file}\"\n      field: \"log.file.path\"\n      target_prefix: \"\"", "custom": "pipeline: suricata.common" } } } } } }'
-echo
-
-# Import - Zeek logs
-echo
-echo "Setting up Zeek import package policy..."
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "name": "import-zeek-logs", "description": "Zeek Import logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": ["/nsm/import/*/zeek/logs/*.log"], "data_stream.dataset": "import", "tags": [], "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/zeek/logs/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- script:\n      lang: javascript\n      source: >\n        function process(event) {\n          var pl = event.Get(\"import.file\").slice(0,-4);\n          event.Put(\"@metadata.pipeline\", \"zeek.\" + pl);\n        }\n- add_fields:\n    target: event\n    fields:\n      category: network\n      module: zeek\n      imported: true\n- add_tags:\n    tags: \"ics\"\n    when:\n      regexp:\n        import.file: \"^bacnet*|^bsap*|^cip*|^cotp*|^dnp3*|^ecat*|^enip*|^modbus*|^opcua*|^profinet*|^s7comm*\"", "custom": "exclude_files: [\"broker|capture_loss|ecat_arp_info|loaded_scripts|packet_filter|stats|stderr|stdout.log$\"]\n" } } } } } }'
-echo
-
-# Strelka logs
-echo
-echo "Setting up Strelka package policy..."
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "strelka-logs", "name": "strelka-logs", "description": "Strelka logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/strelka/log/strelka.log" ], "data_stream.dataset": "strelka", "tags": [],"processors": "- add_fields:\n    target: event\n    fields:\n      category: file\n      module: strelka", "custom": "pipeline: strelka.file" }}}}}}'
-echo
-
-# Syslog TCP Port 514
-echo
-echo "Setting up Syslog TCP package policy..."
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{ "policy_id": "so-grid-nodes", "package": { "name": "tcp", "version": "1.5.0" }, "id": "syslog-tcp-514", "name": "syslog-tcp-514", "description": "Syslog Over TCP Port 514", "namespace": "so", "inputs": { "tcp-tcp": { "enabled": true, "streams": { "tcp.generic": { "enabled": true, "vars": { "listen_address": "0.0.0.0", "listen_port": "514", "data_stream.dataset": "syslog", "pipeline": "syslog", "processors": "- add_fields:\n    target: event\n    fields:\n      module: syslog", "tags": [ "syslog" ], "syslog_options": "field: message\n#format: auto\n#timezone: Local" } } } } } }'
-echo
-
-# Syslog UDP Port 514
-echo
-echo "Setting up Syslog UDP package policy..."
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{ "policy_id": "so-grid-nodes", "package": { "name": "udp", "version": "1.5.0" }, "id": "syslog-udp-514", "name": "syslog-udp-514", "description": "Syslog over UDP Port 514", "namespace": "so", "inputs": { "udp-udp": { "enabled": true, "streams": { "udp.generic": { "enabled": true, "vars": { "listen_address": "0.0.0.0", "listen_port": "514", "data_stream.dataset": "syslog", "pipeline": "syslog", "max_message_size": "10KiB", "keep_null": false, "processors": "- add_fields:\n    target: event\n    fields: \n      module: syslog\n", "tags": [ "syslog" ], "syslog_options": "field: message\n#format: auto\n#timezone: Local" } } } } } }'
-echo
-
-# Kratos logs
-echo
-echo "Setting up Kratos package policy..."
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "kratos-logs", "name": "kratos-logs", "description": "Kratos logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/opt/so/log/kratos/kratos.log" ], "data_stream.dataset": "kratos", "tags": [],"custom":"pipeline: kratos","processors": "- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n    add_error_key: true      \n- add_fields:\n    target: event\n    fields:\n      category: iam\n      module: kratos" }}}}}}'
-echo
-
-# RITA Logs
-#echo
-#echo "Setting up RITA package policy..."
-#curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "rita-logs", "name": "rita-logs", "description": "RITA Beacon logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/rita/beacons.csv", "/nsm/rita/long-connections.csv", "/nsm/rita/short-connections.csv", "/nsm/rita/exploded-dns.csv" ], "data_stream.dataset": "rita", "tags": [], "processors": "- add_fields:\n    target: event\n    fields:\n      category: network\n      module: rita\n- if:\n    log.file.path: beacons.csv\n  then: \n    - add_fields:\n        target: \"@metadata\"\n        fields:\n          pipeline: rita.beacon\n- if:\n    regexp:\n      log.file.path: \"*connections.csv\"\n  then: \n    - add_fields:\n        target: \"@metadata\"\n        fields:\n          pipeline: rita.connection\n- if:\n    log.file.path: \"exploded-dns.csv\"\n  then: \n    - add_fields:\n        target: \"@metadata\"\n        fields:\n          pipeline: rita.dns" }}}}}}'
-#echo
-
-# Elasticsearch logs
-echo
-echo "Setting up Elasticsearch package policy..."
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "elasticsearch", "version": "1.0.0" }, "id": "elasticsearch-logs", "name": "elasticsearch-logs", "description": "Elasticsearch Logs", "namespace": "default", "inputs": { "elasticsearch-logfile": { "enabled": true, "streams": { "elasticsearch.audit": { "enabled": false, "vars": { "paths": [ "/var/log/elasticsearch/*_audit.json" ] } }, "elasticsearch.deprecation": { "enabled": false, "vars": { "paths": [ "/var/log/elasticsearch/*_deprecation.json" ] } }, "elasticsearch.gc": { "enabled": false, "vars": { "paths": [ "/var/log/elasticsearch/gc.log.[0-9]*", "/var/log/elasticsearch/gc.log" ] } }, "elasticsearch.server": { "enabled": true, "vars": { "paths": [ "/opt/so/log/elasticsearch/*.log" ] } }, "elasticsearch.slowlog": { "enabled": false, "vars": { "paths": [ "/var/log/elasticsearch/*_index_search_slowlog.json", "/var/log/elasticsearch/*_index_indexing_slowlog.json" ] } } } }, "elasticsearch-elasticsearch/metrics": { "enabled": false, "vars": { "hosts": [ "http://localhost:9200" ], "scope": "node" }, "streams": { "elasticsearch.stack_monitoring.ccr": { "enabled": false }, "elasticsearch.stack_monitoring.cluster_stats": { "enabled": false }, "elasticsearch.stack_monitoring.enrich": { "enabled": false }, "elasticsearch.stack_monitoring.index": { "enabled": false }, "elasticsearch.stack_monitoring.index_recovery": { "enabled": false, "vars": { "active.only": true } }, "elasticsearch.stack_monitoring.index_summary": { "enabled": false }, "elasticsearch.stack_monitoring.ml_job": { "enabled": false }, "elasticsearch.stack_monitoring.node": { "enabled": false }, "elasticsearch.stack_monitoring.node_stats": { "enabled": false }, "elasticsearch.stack_monitoring.pending_tasks": { "enabled": false }, "elasticsearch.stack_monitoring.shard": { "enabled": false } } } } }'
-echo
-
-# Logstash logs
-#echo
-#echo "Setting up Logstash package policy..."
-#curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "logstash", "version": "2.0.0" }, "id": "logstash-logs", "name": "logstash-logs", "description": "Logstash logs", "namespace": "default", "inputs": { "logstash-logfile": { "enabled": true, "streams": { "logstash.log": { "enabled": true, "vars": { "paths": [ "/opt/so/logs/logstash/logstash.log" ] } }, "logstash.slowlog": { "enabled": false, "vars": { "paths": [ "/var/log/logstash/logstash-slowlog-plain*.log", "/var/log/logstash/logstash-slowlog-json*.log" ] } } } }, "logstash-logstash/metrics": { "enabled": false, "vars": { "hosts": [ "http://localhost:9600" ], "period": "10s" }, "streams": { "logstash.stack_monitoring.node": { "enabled": false }, "logstash.stack_monitoring.node_stats": { "enabled": false } } } } }'
-#echo
-
-# Kibana logs
-#echo
-#echo "Setting up Kibana package policy..."
-#curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "kibana", "version": "2.0.0" }, "id": "kibana-logs", "name": "kibana-logs", "description": "Kibana logs", "namespace": "default", "inputs": { "kibana-logfile": { "enabled": true, "streams": { "kibana.audit": { "enabled": false, "vars": { "paths": [ "/opt/so/log/kibana/kibana.log" ] } }, "kibana.log": { "enabled": true, "vars": { "paths": [ "/opt/so/log/kibana/kibana.log" ] } } } }, "kibana-kibana/metrics": { "enabled": false, "vars": { "hosts": [ "http://localhost:5601" ] }, "streams": { "kibana.stack_monitoring.cluster_actions": { "enabled": false }, "kibana.stack_monitoring.cluster_rules": { "enabled": false }, "kibana.stack_monitoring.node_actions": { "enabled": false }, "kibana.stack_monitoring.node_rules": { "enabled": false }, "kibana.stack_monitoring.stats": { "enabled": false }, "kibana.stack_monitoring.status": { "enabled": false } } } } }'
-#echo
-
-# Redis logs
-echo
-echo "Setting up Redis package policy..."
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "redis", "version": "1.4.0" }, "id": "redis-logs", "name": "redis-logs", "description": "Redis logs", "namespace": "default", "inputs": { "redis-logfile": { "enabled": true, "streams": { "redis.log": { "enabled": true, "vars": { "paths": [ "/opt/so/log/redis/redis.log" ], "tags": [ "redis-log" ], "preserve_original_event": false } } } }, "redis-redis": { "enabled": false, "streams": { "redis.slowlog": { "enabled": false, "vars": { "hosts": [ "127.0.0.1:6379" ], "password": "" } } } }, "redis-redis/metrics": { "enabled": false, "vars": { "hosts": [ "127.0.0.1:6379" ], "idle_timeout": "20s", "maxconn": 10, "network": "tcp", "password": "" }, "streams": { "redis.info": { "enabled": false, "vars": { "period": "10s" } }, "redis.key": { "enabled": false, "vars": { "key.patterns": "- limit: 20\n  pattern: '*'\n", "period": "10s" } }, "redis.keyspace": { "enabled": false, "vars": { "period": "10s" } } } } } }'
-echo
-
-# IDH logs
-echo
-echo "Setting up IDH package policy..."
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{"policy_id":"so-grid-nodes","package":{"name":"log","version":"1.1.1"},"id":"idh-logs","name":"idh-logs","namespace":"so","description":"IDH integration","inputs":{"logs-logfile":{"enabled":true,"streams":{"log.log":{"enabled":true,"vars":{"paths":["/nsm/idh/opencanary.log"],"data_stream.dataset":"idh","custom":"pipeline: common","processors": "\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n    add_error_key: true\n- drop_fields:\n    when:\n      equals:\n        logtype: \"1001\"\n    fields: [\"src_host\", \"src_port\", \"dst_host\", \"dst_port\" ]\n    ignore_missing: true\n- rename:\n    fields:\n      - from: \"src_host\"\n        to: \"source.ip\"\n      - from: \"src_port\"\n        to: \"source.port\"\n      - from: \"dst_host\"\n        to: \"destination.host\"\n      - from: \"dst_port\"\n        to: \"destination.port\"\n    ignore_missing: true\n- convert:\n    fields:\n      - {from: \"logtype\", to: \"event.code\", type: \"string\"}\n    ignore_missing: true\n- drop_fields:\n    fields: '\''[\"prospector\", \"input\", \"offset\", \"beat\"]'\''\n- add_fields:\n    target: event\n    fields:\n      category: host\n      module: opencanary","tags":[]}}}}}}'
-echo
-
-# SOC - Server logs
-echo
-echo "Setting up SOC - Server Logs package policy..."
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{"package":{"name":"log","version":"1.1.2"},"name":"soc-server-logs","namespace":"so","description":"Security Onion Console Logs","policy_id":"so-grid-nodes","inputs":{"logs-logfile":{"enabled":true,"streams":{"log.log":{"enabled":true,"vars":{"paths":["/opt/so/log/soc/sensoroni-server.log"],"data_stream.dataset":"soc","custom":"pipeline: common","processors": "- decode_json_fields:\n    fields: [\"message\"]\n    target: \"soc\"\n    process_array: true\n    max_depth: 2\n    add_error_key: true      \n- add_fields:\n    target: event\n    fields:\n      category: host\n      module: soc\n      dataset_temp: server\n- rename:\n    fields:\n      - from: \"soc.fields.sourceIp\"\n        to: \"source.ip\"\n      - from: \"soc.fields.status\"\n        to: \"http.response.status_code\"\n      - from: \"soc.fields.method\"\n        to: \"http.request.method\"\n      - from: \"soc.fields.path\"\n        to: \"url.path\"\n      - from: \"soc.message\"\n        to: \"event.action\"\n      - from: \"soc.level\"\n        to: \"log.level\"\n    ignore_missing: true","tags":[]}}}}}}'
-echo
-
-# SOC - Sensoroni logs
-echo
-echo "Setting up SOC - Sensoroni Logs package policy..."
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{"package":{"name":"log","version":"1.1.2"},"name":"soc-sensoroni-logs","namespace":"so","description":"Security Onion - Sensoroni - Logs","policy_id":"so-grid-nodes","inputs":{"logs-logfile":{"enabled":true,"streams":{"log.log":{"enabled":true,"vars":{"paths":["/opt/so/log/sensoroni/sensoroni.log"],"data_stream.dataset":"soc","custom":"pipeline: common","processors": "- decode_json_fields:\n    fields: [\"message\"]\n    target: \"sensoroni\"\n    process_array: true\n    max_depth: 2\n    add_error_key: true      \n- add_fields:\n    target: event\n    fields:\n      category: host\n      module: soc\n      dataset_temp: sensoroni\n- rename:\n    fields:\n      - from: \"sensoroni.fields.sourceIp\"\n        to: \"source.ip\"\n      - from: \"sensoroni.fields.status\"\n        to: \"http.response.status_code\"\n      - from: \"sensoroni.fields.method\"\n        to: \"http.request.method\"\n      - from: \"sensoroni.fields.path\"\n        to: \"url.path\"\n      - from: \"sensoroni.message\"\n        to: \"event.action\"\n      - from: \"sensoroni.level\"\n        to: \"log.level\"\n    ignore_missing: true","tags":[]}}}}}}'
-echo
-
-# SOC - Elastic Auth Sync logs
-echo
-echo "Setting up SOC - Elastic Auth Sync Logs package policy..."
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{"package":{"name":"log","version":"1.1.2"},"name":"soc-auth-sync-logs","namespace":"so","description":"Security Onion - Elastic Auth Sync - Logs","policy_id":"so-grid-nodes","inputs":{"logs-logfile":{"enabled":true,"streams":{"log.log":{"enabled":true,"vars":{"paths":["/opt/so/log/soc/sync.log"],"data_stream.dataset":"soc","custom":"pipeline: common","processors": "- dissect:\n    tokenizer: \"%{event.action}\"\n    field: \"message\"\n    target_prefix: \"\"\n- add_fields:\n    target: event\n    fields:\n      category: host\n      module: soc\n      dataset_temp: auth_sync","tags":[]}}}}}}'
-echo
-
-# SOC - Salt Relay logs
-echo
-echo "Setting up SOC - Salt_Relay Logs package policy..."
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{"package":{"name":"log","version":"1.1.2"},"name":"soc-salt-relay-logs","namespace":"so","description":"Security Onion - Salt Relay - Logs","policy_id":"so-grid-nodes","inputs":{"logs-logfile":{"enabled":true,"streams":{"log.log":{"enabled":true,"vars":{"paths":["/opt/so/log/soc/salt-relay.log"],"data_stream.dataset":"soc","custom":"pipeline: common","processors": "- dissect:\n    tokenizer: \"%{soc.ts} | %{event.action}\"\n    field: \"message\"\n    target_prefix: \"\"\n- add_fields:\n    target: event\n    fields:\n      category: host\n      module: soc\n      dataset_temp: salt_relay","tags":[]}}}}}}'
-echo

salt/common/tools/sbin/so-elastic-fleet-setup

@@ -8,40 +8,6 @@
 
 . /usr/sbin/so-common
 
-elastic_fleet_policy_create() {
-
-    NAME=$1
-    DESC=$2
-    FLEETSERVER=$3
-
-    JSON_STRING=$( jq -n \
-                    --arg NAME "$NAME" \
-                    --arg DESC "$DESC" \
-                    --arg FLEETSERVER "$FLEETSERVER" \
-                    '{"name": $NAME,"id":$NAME,"description":$DESC,"namespace":"default","monitoring_enabled":["logs"],"inactivity_timeout":1209600,"has_fleet_server":$FLEETSERVER}'
-                    )
-    # Create Fleet Policy
-    curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/agent_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" 
-
-}
-
-elastic_fleet_policy_update() {
-
-    POLICYID=$1
-    JSON_STRING_UPDATE=$2
-
-    curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/agent_policies/$POLICYID" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING_UPDATE"
-}
-
-elastic_fleet_integration_create() {
-
-    JSONBLOB=$1
-
-    #curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/agent_policies/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSONBLOB"
-    curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSONBLOB"
-
-}
-
 printf "\n### Create ES Token ###\n"
 ESTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/service_tokens" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' |  jq -r .value)
 
@@ -79,30 +45,22 @@ printf "\n\n"
 
 # Manager Fleet Server Host
 elastic_fleet_policy_create "FleetServer_{{ GLOBALS.hostname }}" "Fleet Server - {{ GLOBALS.hostname }}" "true" | jq
-#elastic_fleet_policy_update "FleetServer_{{ GLOBALS.hostname }}" "@/opt/so/saltstack/default/salt/elasticfleet/files/FleeServerHost_Fixup.json"
 
-#Temp Fixup
+#Temp Fixup for ES Output bug
 JSON_STRING=$( jq -n \
                 --arg NAME "FleetServer_{{ GLOBALS.hostname }}" \
                 '{"name": $NAME,"description": $NAME,"namespace":"default","monitoring_enabled":["logs"],"inactivity_timeout":120,"data_output_id":"so-manager_elasticsearch"}'
                 )
 curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/agent_policies/FleetServer_{{ GLOBALS.hostname }}" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
 
+# Initial Endpoints Policy
+elastic_fleet_policy_create "endpoints-initial" "Initial Endpoint Policy" "false"
 
-# Initial Endpoints 
+# Grid Nodes Policy
-elastic_fleet_policy_create "endpoints-initial" "Initial Endpoint Policy" "false" | jq
-for INTEGRATION in /opt/so/saltstack/default/salt/elasticfleet/files/integrations/endpoints-initial/*.json
-do
-  elastic_fleet_integration_create "@$INTEGRATION" | jq
-done
-
-# Grid Nodes
 elastic_fleet_policy_create "so-grid-nodes" "SO Grid Node Policy" "false"
-for INTEGRATION in /opt/so/saltstack/default/salt/elasticfleet/files/integrations/grid-nodes/*.json
-do
-  elastic_fleet_integration_create "@$INTEGRATION" | jq
-done
 
+# Load Integrations for default policies
+so-elastic-fleet-integration-policy-load
 
 ### Finalization ###
 

salt/common/tools/sbin/so-minion

@@ -239,6 +239,22 @@ function create_fleet_policy() {
 	curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/agent_policies/FleetServer_$LSHOSTNAME" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING_UPDATE"
 }
 
+function update_fleet_host_urls() {
+	# Query for current Fleet Host URLs & append New Fleet Node Hostname & IP
+    JSON_STRING=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/fleet_server_hosts' | jq --arg HOSTNAME "https://$LSHOSTNAME:8220" --arg IP "https://$MAINIP:8220" -r '.items[].host_urls += [ $HOSTNAME, $IP ] | {"name":"Default","host_urls": .items[].host_urls,"is_default":true,"proxy_id":null}')
+
+	# Update Fleet Host URLs
+	curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/fleet_server_hosts/fleet-default-fleet-server-host" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
+}
+
+function update_logstash_outputs() {
+	# Query for current Logstash outputs & append New Fleet Node Hostname & IP
+    JSON_STRING=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/outputs/so-manager_logstash' | jq --arg HOSTNAME "$LSHOSTNAME:5055" --arg IP "$MAINIP:5055" -r '.item.hosts += [ $HOSTNAME, $IP ] | {"name":"grid-logstash","type":"logstash","hosts": .item.hosts,"is_default":true,"is_default_monitoring":true,"config_yaml":""}')
+
+	# Update Logstash Outputs
+    curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/outputs/so-manager_logstash" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
+}
+
 function updateMine() {
 	salt "$MINION_ID" mine.send network.ip_addrs interface="$MNIC"
 }
@@ -255,6 +271,8 @@ function createFLEET() {
 	add_fleet_to_minion
 	add_logstash_to_minion
 	create_fleet_policy
+	update_fleet_host_urls
+	update_logstash_outputs
 }
 
 function createIDH() {

salt/elasticfleet/files/integrations/endpoints-initial/system-endpoints.json

@@ -0,0 +1,76 @@
+{
+  "package": {
+    "name": "system",
+    "version": "1.25.2"
+  },
+  "name": "system-endpoints",
+  "namespace": "default",
+  "policy_id": "endpoints-initial",
+  "inputs": {
+    "system-logfile": {
+      "enabled": true,
+      "streams": {
+        "system.auth": {
+          "enabled": true,
+          "vars": {
+            "ignore_older": "72h",
+            "paths": [
+              "/var/log/auth.log*",
+              "/var/log/secure*"
+            ],
+            "preserve_original_event": false,
+            "tags": [
+              "system-auth"
+            ]
+          }
+        },
+        "system.syslog": {
+          "enabled": true,
+          "vars": {
+            "paths": [
+              "/var/log/messages*",
+              "/var/log/syslog*"
+            ],
+            "tags": [],
+            "ignore_older": "72h"
+          }
+        }
+      }
+    },
+    "system-winlog": {
+      "enabled": true,
+      "streams": {
+        "system.application": {
+          "enabled": true,
+          "vars": {
+            "preserve_original_event": false,
+            "ignore_older": "72h",
+            "language": 0,
+            "tags": []
+          }
+        },
+        "system.security": {
+          "enabled": true,
+          "vars": {
+            "preserve_original_event": false,
+            "ignore_older": "72h",
+            "language": 0,
+            "tags": []
+          }
+        },
+        "system.system": {
+          "enabled": true,
+          "vars": {
+            "preserve_original_event": false,
+            "ignore_older": "72h",
+            "language": 0,
+            "tags": []
+          }
+        }
+      }
+    },
+    "system-system/metrics": {
+      "enabled": false
+    }
+  }
+}

salt/elasticfleet/files/integrations/endpoints-initial/windows-endpoints.json

@@ -0,0 +1,59 @@
+{
+  "package": {
+    "name": "windows",
+    "version": "1.19.1"
+  },
+  "name": "windows-endpoints",
+  "namespace": "default",
+  "policy_id": "endpoints-initial",
+  "inputs": {
+    "windows-winlog": {
+      "enabled": true,
+      "streams": {
+        "windows.forwarded": {
+          "enabled": true,
+          "vars": {
+            "preserve_original_event": false,
+            "ignore_older": "72h",
+            "language": 0,
+            "tags": [
+              "forwarded"
+            ]
+          }
+        },
+        "windows.powershell": {
+          "enabled": true,
+          "vars": {
+            "preserve_original_event": false,
+            "event_id": "400, 403, 600, 800",
+            "ignore_older": "72h",
+            "language": 0,
+            "tags": []
+          }
+        },
+        "windows.powershell_operational": {
+          "enabled": true,
+          "vars": {
+            "preserve_original_event": false,
+            "event_id": "4103, 4104, 4105, 4106",
+            "ignore_older": "72h",
+            "language": 0,
+            "tags": []
+          }
+        },
+        "windows.sysmon_operational": {
+          "enabled": true,
+          "vars": {
+            "preserve_original_event": false,
+            "ignore_older": "72h",
+            "language": 0,
+            "tags": []
+          }
+        }
+      }
+    },
+    "windows-windows/metrics": {
+      "enabled": false
+    }
+  }
+}

salt/elasticfleet/files/integrations/grid-nodes/elasticsearch-logs.json

@@ -0,0 +1,106 @@
+{
+  "package": {
+    "name": "elasticsearch",
+    "version": "1.0.0"
+  },
+  "name": "elasticsearch-logs",
+  "namespace": "default",
+  "description": "Elasticsearch Logs",
+  "policy_id": "so-grid-nodes",
+  "inputs": {
+    "elasticsearch-logfile": {
+      "enabled": true,
+      "streams": {
+        "elasticsearch.audit": {
+          "enabled": false,
+          "vars": {
+            "paths": [
+              "/var/log/elasticsearch/*_audit.json"
+            ]
+          }
+        },
+        "elasticsearch.deprecation": {
+          "enabled": false,
+          "vars": {
+            "paths": [
+              "/var/log/elasticsearch/*_deprecation.json"
+            ]
+          }
+        },
+        "elasticsearch.gc": {
+          "enabled": false,
+          "vars": {
+            "paths": [
+              "/var/log/elasticsearch/gc.log.[0-9]*",
+              "/var/log/elasticsearch/gc.log"
+            ]
+          }
+        },
+        "elasticsearch.server": {
+          "enabled": true,
+          "vars": {
+            "paths": [
+              "/opt/so/log/elasticsearch/*.log"
+            ]
+          }
+        },
+        "elasticsearch.slowlog": {
+          "enabled": false,
+          "vars": {
+            "paths": [
+              "/var/log/elasticsearch/*_index_search_slowlog.json",
+              "/var/log/elasticsearch/*_index_indexing_slowlog.json"
+            ]
+          }
+        }
+      }
+    },
+    "elasticsearch-elasticsearch/metrics": {
+      "enabled": false,
+      "vars": {
+        "hosts": [
+          "http://localhost:9200"
+        ],
+        "scope": "node"
+      },
+      "streams": {
+        "elasticsearch.stack_monitoring.ccr": {
+          "enabled": false
+        },
+        "elasticsearch.stack_monitoring.cluster_stats": {
+          "enabled": false
+        },
+        "elasticsearch.stack_monitoring.enrich": {
+          "enabled": false
+        },
+        "elasticsearch.stack_monitoring.index": {
+          "enabled": false
+        },
+        "elasticsearch.stack_monitoring.index_recovery": {
+          "enabled": false,
+          "vars": {
+            "active.only": true
+          }
+        },
+        "elasticsearch.stack_monitoring.index_summary": {
+          "enabled": false
+        },
+        "elasticsearch.stack_monitoring.ml_job": {
+          "enabled": false
+        },
+        "elasticsearch.stack_monitoring.node": {
+          "enabled": false
+        },
+        "elasticsearch.stack_monitoring.node_stats": {
+          "enabled": false
+        },
+        "elasticsearch.stack_monitoring.pending_tasks": {
+          "enabled": false
+        },
+        "elasticsearch.stack_monitoring.shard": {
+          "enabled": false
+        }
+      }
+    }
+  }
+}

salt/elasticfleet/files/integrations/grid-nodes/import-evtx-logs.json

@@ -0,0 +1,29 @@
+{
+  "package": {
+    "name": "log",
+    "version": "1.1.0"
+  },
+  "name": "import-evtx-logs",
+  "namespace": "so",
+  "description": "Import Windows EVTX logs",
+  "policy_id": "so-grid-nodes",
+  "inputs": {
+    "logs-logfile": {
+      "enabled": true,
+      "streams": {
+        "log.log": {
+          "enabled": true,
+          "vars": {
+            "paths": [
+              "/nsm/import/*/evtx/data.json"
+            ],
+            "data_stream.dataset": "import",
+            "tags": [],
+            "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- add_fields:\n    target: event\n    fields:\n      module: windows_eventlog\n      imported: true",
+            "custom": "pipeline: import.wel"
+          }
+        }
+      }
+    }
+  }
+}

salt/elasticfleet/files/integrations/grid-nodes/import-suricata-logs.json

@@ -0,0 +1,29 @@
+{
+  "package": {
+    "name": "log",
+    "version": "1.1.0"
+  },
+  "name": "import-suricata-logs",
+  "namespace": "so",
+  "description": "Import Suricata logs",
+  "policy_id": "so-grid-nodes",
+  "inputs": {
+    "logs-logfile": {
+      "enabled": true,
+      "streams": {
+        "log.log": {
+          "enabled": true,
+          "vars": {
+            "paths": [
+              "/nsm/import/*/suricata/eve*.json"
+            ],
+            "data_stream.dataset": "import",
+            "tags": [],
+            "processors": "- add_fields:\n    target: event\n    fields:\n      category: network\n      module: suricata\n      imported: true\n- dissect:\n      tokenizer: \"/nsm/import/%{import.id}/suricata/%{import.file}\"\n      field: \"log.file.path\"\n      target_prefix: \"\"",
+            "custom": "pipeline: suricata.common"
+          }
+        }
+      }
+    }
+  }
+}

salt/elasticfleet/files/integrations/grid-nodes/import-zeek-logs.json

@@ -0,0 +1,29 @@
+{
+  "package": {
+    "name": "log",
+    "version": "1.1.0"
+  },
+  "name": "import-zeek-logs",
+  "namespace": "so",
+  "description": "Zeek Import logs",
+  "policy_id": "so-grid-nodes",
+  "inputs": {
+    "logs-logfile": {
+      "enabled": true,
+      "streams": {
+        "log.log": {
+          "enabled": true,
+          "vars": {
+            "paths": [
+              "/nsm/import/*/zeek/logs/*.log"
+            ],
+            "data_stream.dataset": "import",
+            "tags": [],
+            "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/zeek/logs/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- script:\n      lang: javascript\n      source: >\n        function process(event) {\n          var pl = event.Get(\"import.file\").slice(0,-4);\n          event.Put(\"@metadata.pipeline\", \"zeek.\" + pl);\n        }\n- add_fields:\n    target: event\n    fields:\n      category: network\n      module: zeek\n      imported: true\n- add_tags:\n    tags: \"ics\"\n    when:\n      regexp:\n        import.file: \"^bacnet*|^bsap*|^cip*|^cotp*|^dnp3*|^ecat*|^enip*|^modbus*|^opcua*|^profinet*|^s7comm*\"",
+            "custom": "exclude_files: [\"broker|capture_loss|ecat_arp_info|loaded_scripts|packet_filter|stats|stderr|stdout.log$\"]\n"
+          }
+        }
+      }
+    }
+  }
+}

salt/elasticfleet/files/integrations/grid-nodes/osquery-grid-nodes.json

@@ -0,0 +1,20 @@
+{
+  "package": {
+    "name": "osquery_manager",
+    "version": "1.6.0"
+  },
+  "name": "osquery-grid-nodes",
+  "namespace": "default",
+  "policy_id": "so-grid-nodes",
+  "inputs": {
+    "osquery_manager-osquery": {
+      "enabled": true,
+      "streams": {
+        "osquery_manager.result": {
+          "enabled": true,
+          "vars": {}
+        }
+      }
+    }
+  }
+}

salt/elasticfleet/files/integrations/grid-nodes/redis-logs.json

@@ -0,0 +1,76 @@
+{
+  "package": {
+    "name": "redis",
+    "version": "1.4.0"
+  },
+  "name": "redis-logs",
+  "namespace": "default",
+  "description": "Redis logs",
+  "policy_id": "so-grid-nodes",
+  "inputs": {
+    "redis-logfile": {
+      "enabled": true,
+      "streams": {
+        "redis.log": {
+          "enabled": true,
+          "vars": {
+            "paths": [
+              "/opt/so/log/redis/redis.log"
+            ],
+            "tags": [
+              "redis-log"
+            ],
+            "preserve_original_event": false
+          }
+        }
+      }
+    },
+    "redis-redis": {
+      "enabled": false,
+      "streams": {
+        "redis.slowlog": {
+          "enabled": false,
+          "vars": {
+            "hosts": [
+              "127.0.0.1:6379"
+            ],
+            "password": ""
+          }
+        }
+      }
+    },
+    "redis-redis/metrics": {
+      "enabled": false,
+      "vars": {
+        "hosts": [
+          "127.0.0.1:6379"
+        ],
+        "idle_timeout": "20s",
+        "maxconn": 10,
+        "network": "tcp",
+        "password": ""
+      },
+      "streams": {
+        "redis.info": {
+          "enabled": false,
+          "vars": {
+            "period": "10s"
+          }
+        },
+        "redis.key": {
+          "enabled": false,
+          "vars": {
+            "key.patterns": "- limit: 20\n  pattern: *\n",
+            "period": "10s"
+          }
+        },
+        "redis.keyspace": {
+          "enabled": false,
+          "vars": {
+            "period": "10s"
+          }
+        }
+      }
+    }
+  }
+}

salt/elasticfleet/files/integrations/grid-nodes/soc-auth-sync-logs.json

@@ -0,0 +1,29 @@
+{
+  "package": {
+    "name": "log",
+    "version": "1.1.2"
+  },
+  "name": "soc-auth-sync-logs",
+  "namespace": "so",
+  "description": "Security Onion - Elastic Auth Sync - Logs",
+  "policy_id": "so-grid-nodes",
+  "inputs": {
+    "logs-logfile": {
+      "enabled": true,
+      "streams": {
+        "log.log": {
+          "enabled": true,
+          "vars": {
+            "paths": [
+              "/opt/so/log/soc/sync.log"
+            ],
+            "data_stream.dataset": "soc",
+            "tags": [],
+            "processors": "- dissect:\n    tokenizer: \"%{event.action}\"\n    field: \"message\"\n    target_prefix: \"\"\n- add_fields:\n    target: event\n    fields:\n      category: host\n      module: soc\n      dataset_temp: auth_sync",
+            "custom": "pipeline: common"
+          }
+        }
+      }
+    }
+  }
+}

salt/elasticfleet/files/integrations/grid-nodes/soc-salt-relay-logs.json

@@ -0,0 +1,29 @@
+{
+  "package": {
+    "name": "log",
+    "version": "1.1.2"
+  },
+  "name": "soc-salt-relay-logs",
+  "namespace": "so",
+  "description": "Security Onion - Salt Relay - Logs",
+  "policy_id": "so-grid-nodes",
+  "inputs": {
+    "logs-logfile": {
+      "enabled": true,
+      "streams": {
+        "log.log": {
+          "enabled": true,
+          "vars": {
+            "paths": [
+              "/opt/so/log/soc/salt-relay.log"
+            ],
+            "data_stream.dataset": "soc",
+            "tags": [],
+            "processors": "- dissect:\n    tokenizer: \"%{soc.ts} | %{event.action}\"\n    field: \"message\"\n    target_prefix: \"\"\n- add_fields:\n    target: event\n    fields:\n      category: host\n      module: soc\n      dataset_temp: salt_relay",
+            "custom": "pipeline: common"
+          }
+        }
+      }
+    }
+  }
+}

salt/elasticfleet/files/integrations/grid-nodes/soc-sensoroni-logs.json

@@ -0,0 +1,29 @@
+{
+  "package": {
+    "name": "log",
+    "version": "1.1.2"
+  },
+  "name": "soc-sensoroni-logs",
+  "namespace": "so",
+  "description": "Security Onion - Sensoroni - Logs",
+  "policy_id": "so-grid-nodes",
+  "inputs": {
+    "logs-logfile": {
+      "enabled": true,
+      "streams": {
+        "log.log": {
+          "enabled": true,
+          "vars": {
+            "paths": [
+              "/opt/so/log/sensoroni/sensoroni.log"
+            ],
+            "data_stream.dataset": "soc",
+            "tags": [],
+            "processors": "- decode_json_fields:\n    fields: [\"message\"]\n    target: \"sensoroni\"\n    process_array: true\n    max_depth: 2\n    add_error_key: true      \n- add_fields:\n    target: event\n    fields:\n      category: host\n      module: soc\n      dataset_temp: sensoroni\n- rename:\n    fields:\n      - from: \"sensoroni.fields.sourceIp\"\n        to: \"source.ip\"\n      - from: \"sensoroni.fields.status\"\n        to: \"http.response.status_code\"\n      - from: \"sensoroni.fields.method\"\n        to: \"http.request.method\"\n      - from: \"sensoroni.fields.path\"\n        to: \"url.path\"\n      - from: \"sensoroni.message\"\n        to: \"event.action\"\n      - from: \"sensoroni.level\"\n        to: \"log.level\"\n    ignore_missing: true",
+            "custom": "pipeline: common"
+          }
+        }
+      }
+    }
+  }
+}

salt/elasticfleet/files/integrations/grid-nodes/soc-server-logs.json

@@ -0,0 +1,29 @@
+{
+  "package": {
+    "name": "log",
+    "version": "1.1.2"
+  },
+  "name": "soc-server-logs",
+  "namespace": "so",
+  "description": "Security Onion Console Logs",
+  "policy_id": "so-grid-nodes",
+  "inputs": {
+    "logs-logfile": {
+      "enabled": true,
+      "streams": {
+        "log.log": {
+          "enabled": true,
+          "vars": {
+            "paths": [
+              "/opt/so/log/soc/sensoroni-server.log"
+            ],
+            "data_stream.dataset": "soc",
+            "tags": [],
+            "processors": "- decode_json_fields:\n    fields: [\"message\"]\n    target: \"soc\"\n    process_array: true\n    max_depth: 2\n    add_error_key: true      \n- add_fields:\n    target: event\n    fields:\n      category: host\n      module: soc\n      dataset_temp: server\n- rename:\n    fields:\n      - from: \"soc.fields.sourceIp\"\n        to: \"source.ip\"\n      - from: \"soc.fields.status\"\n        to: \"http.response.status_code\"\n      - from: \"soc.fields.method\"\n        to: \"http.request.method\"\n      - from: \"soc.fields.path\"\n        to: \"url.path\"\n      - from: \"soc.message\"\n        to: \"event.action\"\n      - from: \"soc.level\"\n        to: \"log.level\"\n    ignore_missing: true",
+            "custom": "pipeline: common"
+          }
+        }
+      }
+    }
+  }
+}

salt/elasticfleet/files/integrations/grid-nodes/strelka-logs.json

@@ -0,0 +1,29 @@
+{
+  "package": {
+    "name": "log",
+    "version": "1.1.0"
+  },
+  "name": "strelka-logs",
+  "namespace": "so",
+  "description": "Strelka logs",
+  "policy_id": "so-grid-nodes",
+  "inputs": {
+    "logs-logfile": {
+      "enabled": true,
+      "streams": {
+        "log.log": {
+          "enabled": true,
+          "vars": {
+            "paths": [
+              "/nsm/strelka/log/strelka.log"
+            ],
+            "data_stream.dataset": "strelka",
+            "tags": [],
+            "processors": "- add_fields:\n    target: event\n    fields:\n      category: file\n      module: strelka",
+            "custom": "pipeline: strelka.file"
+          }
+        }
+      }
+    }
+  }
+}

salt/elasticfleet/files/integrations/grid-nodes/suricata-logs.json

@@ -0,0 +1,29 @@
+{
+  "package": {
+    "name": "log",
+    "version": "1.1.0"
+  },
+  "name": "suricata-logs",
+  "namespace": "so",
+  "description": "Suricata integration",
+  "policy_id": "so-grid-nodes",
+  "inputs": {
+    "logs-logfile": {
+      "enabled": true,
+      "streams": {
+        "log.log": {
+          "enabled": true,
+          "vars": {
+            "paths": [
+              "/nsm/suricata/eve*.json"
+            ],
+            "data_stream.dataset": "suricata",
+            "tags": [],
+            "processors": "- add_fields:\n    target: event\n    fields:\n      category: network\n      module: suricata",
+            "custom": "pipeline: suricata.common"
+          }
+        }
+      }
+    }
+  }
+}

salt/elasticfleet/files/integrations/grid-nodes/syslog-tcp-514.json

@@ -0,0 +1,32 @@
+{
+  "package": {
+    "name": "tcp",
+    "version": "1.5.0"
+  },
+  "name": "syslog-tcp-514",
+  "namespace": "so",
+  "description": "Syslog Over TCP Port 514",
+  "policy_id": "so-grid-nodes",
+  "inputs": {
+    "tcp-tcp": {
+      "enabled": true,
+      "streams": {
+        "tcp.generic": {
+          "enabled": true,
+          "vars": {
+            "listen_address": "0.0.0.0",
+            "listen_port": "514",
+            "data_stream.dataset": "syslog",
+            "pipeline": "syslog",
+            "processors": "- add_fields:\n    target: event\n    fields:\n      module: syslog",
+            "tags": [
+              "syslog"
+            ],
+            "syslog_options": "field: message\n#format: auto\n#timezone: Local",
+            "ssl": ""
+          }
+        }
+      }
+    }
+  }
+}

salt/elasticfleet/files/integrations/grid-nodes/syslog-udp-514.json

@@ -0,0 +1,33 @@
+{
+  "package": {
+    "name": "udp",
+    "version": "1.5.0"
+  },
+  "name": "syslog-udp-514",
+  "namespace": "so",
+  "description": "Syslog over UDP Port 514",
+  "policy_id": "so-grid-nodes",
+  "inputs": {
+    "udp-udp": {
+      "enabled": true,
+      "streams": {
+        "udp.generic": {
+          "enabled": true,
+          "vars": {
+            "listen_address": "0.0.0.0",
+            "listen_port": "514",
+            "data_stream.dataset": "syslog",
+            "pipeline": "syslog",
+            "max_message_size": "10KiB",
+            "keep_null": false,
+            "processors": "- add_fields:\n    target: event\n    fields: \n      module: syslog\n",
+            "tags": [
+              "syslog"
+            ],
+            "syslog_options": "field: message\n#format: auto\n#timezone: Local"
+          }
+        }
+      }
+    }
+  }
+}

salt/elasticfleet/files/integrations/grid-nodes/system-grid-nodes.json

@@ -0,0 +1,47 @@
+{
+  "package": {
+    "name": "system",
+    "version": "1.25.2"
+  },
+  "name": "system-grid-nodes",
+  "namespace": "default",
+  "policy_id": "so-grid-nodes",
+  "inputs": {
+    "system-logfile": {
+      "enabled": true,
+      "streams": {
+        "system.auth": {
+          "enabled": true,
+          "vars": {
+            "ignore_older": "72h",
+            "paths": [
+              "/var/log/auth.log*",
+              "/var/log/secure*"
+            ],
+            "preserve_original_event": false,
+            "tags": [
+              "system-auth"
+            ]
+          }
+        },
+        "system.syslog": {
+          "enabled": true,
+          "vars": {
+            "paths": [
+              "/var/log/messages*",
+              "/var/log/syslog*"
+            ],
+            "tags": [],
+            "ignore_older": "72h"
+          }
+        }
+      }
+    },
+    "system-winlog": {
+      "enabled": false
+    },
+    "system-system/metrics": {
+      "enabled": false
+    }
+  }
+}

salt/elasticfleet/files/integrations/grid-nodes/zeek-logs.json

@@ -0,0 +1,29 @@
+{
+  "package": {
+    "name": "log",
+    "version": "1.1.0"
+  },
+  "name": "zeek-logs",
+  "namespace": "so",
+  "description": "Zeek logs",
+  "policy_id": "so-grid-nodes",
+  "inputs": {
+    "logs-logfile": {
+      "enabled": true,
+      "streams": {
+        "log.log": {
+          "enabled": true,
+          "vars": {
+            "paths": [
+              "/nsm/zeek/logs/current/*.log"
+            ],
+            "data_stream.dataset": "zeek",
+            "tags": [],
+            "processors": "- dissect:\n    tokenizer: \"/nsm/zeek/logs/current/%{pipeline}.log\"\n    field: \"log.file.path\"\n    trim_chars: \".log\"\n    target_prefix: \"\"\n- script:\n      lang: javascript\n      source: >\n        function process(event) {\n          var pl = event.Get(\"pipeline\");\n          event.Put(\"@metadata.pipeline\", \"zeek.\" + pl);\n        }\n- add_fields:\n    target: event\n    fields:\n      category: network\n      module: zeek\n- add_tags:\n    tags: \"ics\"\n    when:\n      regexp:\n        pipeline: \"^bacnet*|^bsap*|^cip*|^cotp*|^dnp3*|^ecat*|^enip*|^modbus*|^opcua*|^profinet*|^s7comm*\"",
+            "custom": "exclude_files: [\"broker|capture_loss|ecat_arp_info|loaded_scripts|packet_filter|stats|stderr|stdout.log$\"]\n"
+          }
+        }
+      }
+    }
+  }
+}

salt/top.sls

@@ -347,6 +347,7 @@ base:
     - firewall
     - logstash
     - elasticfleet
+    - elasticfleet.install_agent_grid
     - schedule
     - docker_clean
 

setup/so-setup

@@ -634,6 +634,8 @@ if ! [[ -f $install_opt_file ]]; then
 		logCmd "salt-call state.apply -l info manager"
 		logCmd "salt-call state.apply influxdb -l info"
 		logCmd "salt-call state.highstate -l info"
+		title "Setting up Kibana Default Space"
+		logCmd "so-kibana-space-defaults"
 		add_web_user
 		info "Restarting SOC to pick up initial user"
 		logCmd "so-soc-restart"