Merge pull request #9553 from Security-Onion-Solutions/feature/filebeat_to_elastic_agent_conversion

Initial Conversion of Filebeat Inputs to Elastic Agent Inputs

salt/common/tools/sbin/so-elastic-fleet-agent-policy-delete

@@ -0,0 +1,14 @@
+#/bin/bash
+. /usr/sbin/so-common
+
+POLICY_ID=$1
+
+# Let's snag a cookie from Kibana
+SESSIONCOOKIE=$(curl -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}')
+
+echo "Deleting agent policy $POLICY_ID..."
+
+# Delete agent policy
+curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/agent_policies/delete" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d"{\"agentPolicyId\": \"$POLICY_ID\"}"
+
+echo

salt/common/tools/sbin/so-elastic-fleet-agent-policy-list

@@ -0,0 +1,12 @@
+#/bin/bash
+. /usr/sbin/so-common
+
+# Let's snag a cookie from Kibana
+SESSIONCOOKIE=$(curl -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}')
+
+echo "Setting up default Security Onion package policies for Elastic Agent..."
+
+# List configured agent policies
+curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/agent_policies" | jq 
+
+echo

salt/common/tools/sbin/so-elastic-fleet-agent-policy-view

@@ -0,0 +1,14 @@
+#/bin/bash
+. /usr/sbin/so-common
+
+POLICY_ID=$1
+
+# Let's snag a cookie from Kibana
+SESSIONCOOKIE=$(curl -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}')
+
+echo "Viewing agent policy $POLICY_ID"
+
+# View agent policy
+curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/agent_policies/$POLICY_ID/full" | jq
+
+echo

salt/common/tools/sbin/so-elastic-fleet-data-streams-list

@@ -0,0 +1,12 @@
+#/bin/bash
+. /usr/sbin/so-common
+
+# Let's snag a cookie from Kibana
+SESSIONCOOKIE=$(curl -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}')
+
+echo "Retrieving data stream information..."
+
+# Retrieve data stream information
+curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/data_streams" | jq 
+
+echo

salt/common/tools/sbin/so-elastic-fleet-integration-policy-delete

@@ -0,0 +1,14 @@
+#/bin/bash
+. /usr/sbin/so-common
+
+POLICY_ID=$1
+
+# Let's snag a cookie from Kibana
+SESSIONCOOKIE=$(curl -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}')
+
+echo "Deleting integration policy $POLICY_ID..."
+
+# List configured package policies
+curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies/delete" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d"{\"packagePolicyIds\": [\"$POLICY_ID\"]}"
+
+echo

salt/common/tools/sbin/so-elastic-fleet-integration-policy-list

@@ -0,0 +1,12 @@
+#/bin/bash
+. /usr/sbin/so-common
+
+# Let's snag a cookie from Kibana
+SESSIONCOOKIE=$(curl -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}')
+
+echo "Setting up default Security Onion package policies for Elastic Agent..."
+
+# List configured package policies
+curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/package_policies" | jq 
+
+echo

salt/common/tools/sbin/so-elastic-fleet-integration-policy-load

@@ -0,0 +1,84 @@
+#/bin/bash
+
+. /usr/sbin/so-common
+
+{%- set ZEEKVER = salt['pillar.get']('global:mdengine', '') %}
+{%- set STRELKAENABLED = salt['pillar.get']('strelka:enabled', '0') %}
+{%- set RITAENABLED = salt['pillar.get']('rita:enabled', False) -%}
+
+wait_for_web_response "http://localhost:5601/api/spaces/space/default" "default" 300 "curl -K /opt/so/conf/elasticsearch/curl.config"
+## This hackery will be removed if using Elastic Auth ##
+
+# Let's snag a cookie from Kibana
+SESSIONCOOKIE=$(curl -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}')
+
+# Disable certain Features from showing up in the Kibana UI
+echo
+echo "Setting up default Security Onion package policies for Elastic Agent..."
+
+# Set up Suricata logs
+echo "Setting up Suricata package policy..."
+curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "suricata-logs", "name": "suricata-logs", "description": "Suricata integration", "namespace": "suricata_so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/suricata/eve*.json" ], "data_stream.dataset": "alert", "tags": [],"processors": "- add_fields:\n    target: event\n    fields:\n      category: network\n      module: suricata", "custom": "pipeline: suricata.common" }}}}}}'
+
+{%- if grains['role'] in ['so-eval', 'so-standalone', 'so-sensor',  'so-helix', 'so-heavynode', 'so-import'] %}
+  {%- if ZEEKVER != 'SURICATA' %}
+    {% import_yaml 'filebeat/defaults.yaml' as FBD with context %}
+    {% set FBCONFIG = salt['pillar.get']('filebeat:zeek_logs_enabled', default=FBD.filebeat, merge=True) %}
+    {%- for LOGNAME in FBCONFIG.zeek_logs_enabled %}
+
+# Import - Suricata
+echo "Settings up Suricata import package policy..."
+curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "import-suricata-logs", "name": "import-suricata-logs", "description": "Import Suricata logs", "namespace": "import_so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": ["/nsm/import/*/suricata/eve*.json"], "data_stream.dataset": "alert", "tags": ["import"], "processors": "- add_fields:\n    target: event\n    fields:\n      category: file\n      module: suricata\n      imported: true\n- dissect:\n      tokenizer: \"/nsm/import/%{import.id}/suricata/%{import.file}\"\n      field: \"log.file.path\"\n      target_prefix: \"\"", "custom": "pipeline: suricata.common" } } } } } }'
+
+# Set up Zeek {{ LOGNAME }} logs
+echo "Setting up Zeek {{ LOGNAME }} package policy..."
+curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "name": "zeek-{{ LOGNAME }}", "description": "Zeek {{ LOGNAME }} logs", "namespace": "zeek_so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/zeek/logs/current/{{ LOGNAME }}.log" ], "data_stream.dataset": "{{ LOGNAME }}", "tags": [], "processors": "- add_fields:\n    target: event\n    fields:\n      category: network\n      module: zeek", "custom": "pipeline: zeek.{{ LOGNAME }}" }}}}}}'
+echo
+
+# Import - Zeek
+echo "Setting up Zeek import {{ LOGNAME }} package policy..."
+curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "import-zeek-{{ LOGNAME }}-logs", "name": "import-zeek-{{ LOGNAME }}-logs", "description": "Import Zeek {{ LOGNAME }} logs", "namespace": "import_so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": ["/nsm/import/*/zeek/logs/{{ LOGNAME }}.log"], "data_stream.dataset": "{{ LOGNAME }}", "tags": ["import"], "- add_fields:\n    target: event\n    fields:\n      category: file\n      module: zeek\n      imported: true\n- dissect:\n      tokenizer: \"/nsm/import/%{import.id}/zeek/logs/%{import.file}\"\n      field: \"log.file.path\"\n      target_prefix: \"\"", "custom": "pipeline: zeek.{{ LOGNAME }}" } } } } } }'
+
+    {%- endfor %}
+  {%- endif %}
+{%- endif %}
+
+# Strelka logs
+echo "Setting up Strelka package policy"
+curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "strelka-logs", "name": "strelka-logs", "description": "Strelka logs", "namespace": "strelka_so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/strelka/log/strelka.log" ], "data_stream.dataset": "file", "tags": [],"processors": "- add_fields:\n    target: event\n    fields:\n      category: file\n      module: strelka", "custom": "pipeline: strelka.file" }}}}}}'
+
+# Syslog TCP Port 514
+echo "Setting up Syslog TCP package policy..."
+curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{ "policy_id": "so-grid-nodes", "package": { "name": "tcp", "version": "1.5.0" }, "id": "syslog-tcp-514", "name": "syslog-tcp-514", "description": "Syslog Over TCP Port 514", "namespace": "syslog_so", "inputs": { "tcp-tcp": { "enabled": true, "streams": { "tcp.generic": { "enabled": true, "vars": { "listen_address": "0.0.0.0", "listen_port": "514", "data_stream.dataset": "syslog", "pipeline": "syslog", "processors": "- add_fields:\n    target: event\n    fields:\n      module: syslog", "tags": [ "syslog" ], "syslog_options": "field: message\n#format: auto\n#timezone: Local" } } } } } }'
+
+# Syslog UDP Port 514
+echo "Setting up Syslog UDP package policy..."
+curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{ "policy_id": "so-grid-nodes", "package": { "name": "udp", "version": "1.5.0" }, "id": "syslog-udp-514", "name": "syslog-udp-514", "description": "Syslog over UDP Port 514", "namespace": "syslog_so", "inputs": { "udp-udp": { "enabled": true, "streams": { "udp.generic": { "enabled": true, "vars": { "listen_address": "0.0.0.0", "listen_port": "514", "data_stream.dataset": "syslog", "pipeline": "syslog", "max_message_size": "10KiB", "keep_null": false, "processors": "- add_fields:\n    target: event\n    fields: \n      module: syslog\n", "tags": [ "syslog" ], "syslog_options": "field: message\n#format: auto\n#timezone: Local" } } } } } }'
+
+# Kratos logs
+echo "Setting up Kratos package policy..."
+curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "kratos-logs", "name": "kratos-logs", "description": "Kratos logs", "namespace": "kratos_so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/opt/so/log/kratos/kratos.log" ], "data_stream.dataset": "access", "tags": [],"processors": "- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n    add_error_key: true\n- rename:\n    fields:\n      - from: \"audience\"\n        to: \"event.dataset\"\n    ignore_missing: true\n- add_fields:\n    when:\n      not: \n        has_fields: ['event.dataset']\n    target: ''\n    fields:\n      event.dataset: access", "custom": "pipeline: kratos" }}}}}}'
+
+# RITA Beacon logs
+echo "Setting up RITA Beacon package policy..."
+curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "rita-beacon-logs", "name": "rita-beacon-logs", "description": "RITA Beacon logs", "namespace": "rita_so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/rita/beacons.csv" ], "data_stream.dataset": "beacon", "tags": [],"processors": "- add_fields:\n    target: event\n    fields:\n      category: network\n      module: rita", "custom": "pipeline: rita.beacon" }}}}}}'
+
+# RITA Connection Logs
+echo "Setting up RITA connection  package policy..."
+curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "rita-connection-logs", "name": "rita-connection-logs", "description": "RITA Connection logs", "namespace": "rita_so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/rita/long-connections.csv" ], "data_stream.dataset": "connections", "tags": [],"processors": "- add_fields:\n    target: event\n    fields:\n      category: network\n      module: rita", "custom": "pipeline: rita.connections" }}}}}}'
+
+# RITA DNS Logs
+echo "Setting up RITA DNS package policy..."
+curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "rita-exploded-dns-logs", "name": "rita-exploded-dns-logs", "description": "RITA Exploded DNS logs", "namespace": "rita_so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/rita/exploded-dns.csv" ], "data_stream.dataset": "exploded_dns", "tags": [],"processors": "- add_fields:\n    target: event\n    fields:\n      category: network\n      module: rita", "custom": "pipeline: rita.dns" }}}}}}'
+
+# Elasticsearch logs
+curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "elasticsearch", "version": "1.0.0" }, "id": "elasticsearch-logs", "name": "elasticsearch-logs", "description": "Elasticsearch Logs", "namespace": "default", "inputs": { "elasticsearch-logfile": { "enabled": true, "streams": { "elasticsearch.audit": { "enabled": false, "vars": { "paths": [ "/var/log/elasticsearch/*_audit.json" ] } }, "elasticsearch.deprecation": { "enabled": false, "vars": { "paths": [ "/var/log/elasticsearch/*_deprecation.json" ] } }, "elasticsearch.gc": { "enabled": false, "vars": { "paths": [ "/var/log/elasticsearch/gc.log.[0-9]*", "/var/log/elasticsearch/gc.log" ] } }, "elasticsearch.server": { "enabled": true, "vars": { "paths": [ "/opt/so/log/elasticsearch/*.log" ] } }, "elasticsearch.slowlog": { "enabled": false, "vars": { "paths": [ "/var/log/elasticsearch/*_index_search_slowlog.json", "/var/log/elasticsearch/*_index_indexing_slowlog.json" ] } } } }, "elasticsearch-elasticsearch/metrics": { "enabled": false, "vars": { "hosts": [ "http://localhost:9200" ], "scope": "node" }, "streams": { "elasticsearch.stack_monitoring.ccr": { "enabled": false }, "elasticsearch.stack_monitoring.cluster_stats": { "enabled": false }, "elasticsearch.stack_monitoring.enrich": { "enabled": false }, "elasticsearch.stack_monitoring.index": { "enabled": false }, "elasticsearch.stack_monitoring.index_recovery": { "enabled": false, "vars": { "active.only": true } }, "elasticsearch.stack_monitoring.index_summary": { "enabled": false }, "elasticsearch.stack_monitoring.ml_job": { "enabled": false }, "elasticsearch.stack_monitoring.node": { "enabled": false }, "elasticsearch.stack_monitoring.node_stats": { "enabled": false }, "elasticsearch.stack_monitoring.pending_tasks": { "enabled": false }, "elasticsearch.stack_monitoring.shard": { "enabled": false } } } } }'
+
+# Logstash logs
+curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "logstash", "version": "2.0.0" }, "id": "logstash-logs", "name": "logstash-logs", "description": "Logstash logs", "namespace": "default", "inputs": { "logstash-logfile": { "enabled": true, "streams": { "logstash.log": { "enabled": true, "vars": { "paths": [ "/opt/so/logs/logstash/logstash.log" ] } }, "logstash.slowlog": { "enabled": false, "vars": { "paths": [ "/var/log/logstash/logstash-slowlog-plain*.log", "/var/log/logstash/logstash-slowlog-json*.log" ] } } } }, "logstash-logstash/metrics": { "enabled": false, "vars": { "hosts": [ "http://localhost:9600" ], "period": "10s" }, "streams": { "logstash.stack_monitoring.node": { "enabled": false }, "logstash.stack_monitoring.node_stats": { "enabled": false } } } } }'
+
+# Kibana logs
+curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "kibana", "version": "2.0.0" }, "id": "kibana-logs", "name": "kibana-logs", "description": "Kibana logs", "namespace": "default", "inputs": { "kibana-logfile": { "enabled": true, "streams": { "kibana.audit": { "enabled": false, "vars": { "paths": [ "/opt/so/log/kibana/kibana.log" ] } }, "kibana.log": { "enabled": true, "vars": { "paths": [ "/opt/so/log/kibana/kibana.log" ] } } } }, "kibana-kibana/metrics": { "enabled": false, "vars": { "hosts": [ "http://localhost:5601" ] }, "streams": { "kibana.stack_monitoring.cluster_actions": { "enabled": false }, "kibana.stack_monitoring.cluster_rules": { "enabled": false }, "kibana.stack_monitoring.node_actions": { "enabled": false }, "kibana.stack_monitoring.node_rules": { "enabled": false }, "kibana.stack_monitoring.stats": { "enabled": false }, "kibana.stack_monitoring.status": { "enabled": false } } } } }'
+
+# Redis logs
+curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "redis", "version": "1.4.0" }, "id": "redis-logs", "name": "redis-logs", "description": "Redis logs", "namespace": "default", "inputs": { "redis-logfile": { "enabled": true, "streams": { "redis.log": { "enabled": true, "vars": { "paths": [ "/opt/so/log/redis/redis.log" ], "tags": [ "redis-log" ], "preserve_original_event": false } } } }, "redis-redis": { "enabled": false, "streams": { "redis.slowlog": { "enabled": false, "vars": { "hosts": [ "127.0.0.1:6379" ], "password": "" } } } }, "redis-redis/metrics": { "enabled": false, "vars": { "hosts": [ "127.0.0.1:6379" ], "idle_timeout": "20s", "maxconn": 10, "network": "tcp", "password": "" }, "streams": { "redis.info": { "enabled": false, "vars": { "period": "10s" } }, "redis.key": { "enabled": false, "vars": { "key.patterns": "- limit: 20\n  pattern: '*'\n", "period": "10s" } }, "redis.keyspace": { "enabled": false, "vars": { "period": "10s" } } } } } }'

salt/common/tools/sbin/so-elastic-fleet-setup

@@ -75,6 +75,9 @@ printf '%s\n'\
 # Call Elastic-Fleet Salt State
 salt-call state.apply elastic-fleet queue=True
 
+# Load Elastic Fleet integrations
+/usr/sbin/so-elastic-fleet-integration-policy-load
+
 # Temp
 wget -P /opt/so/saltstack/default/salt/elastic-fleet/files/elastic-agent/  https://github.com/Security-Onion-Solutions/securityonion-docker-rpm/releases/download/so_elastic-agent-8.4.1/so-elastic-agent-8.4.1-darwin-x86_64.tar.gz
 wget -P /opt/so/saltstack/default/salt/elastic-fleet/files/elastic-agent/ https://github.com/Security-Onion-Solutions/securityonion-docker-rpm/releases/download/so_elastic-agent-8.4.1/so-elastic-agent-8.4.1-linux-x86_64.tar.gz

salt/elasticsearch/defaults.yaml

@@ -1152,8 +1152,9 @@ elasticsearch:
       delete: 365
       index_sorting: False
       index_template:
+        data_stream: {}
         index_patterns:
-          - so-*
+          - logs-*_so
         template:
           mappings:
             dynamic_templates:
@@ -2154,11 +2155,12 @@ elasticsearch:
           - common-settings
           - common-dynamic-mappings
         priority: 500
-    so-ids:
+    so-suricata:
       index_sorting: False
       index_template:
+        data_stream: {}
         index_patterns:
-          - so-ids*
+          - logs-*-suricata_so
         template:
           mappings:
             dynamic_templates:
@@ -2324,8 +2326,9 @@ elasticsearch:
     so-import:
       index_sorting: False
       index_template:
+        data_stream: {}
         index_patterns:
-          - so-import*
+          - logs-*-import_so
         template:
           mappings:
             dynamic_templates:
@@ -4162,8 +4165,9 @@ elasticsearch:
     so-strelka:
       index_sorting: False
       index_template:
+        data_stream: {}
         index_patterns:
-          - so-strelka*
+          - logs-*-strelka_so
         template:
           mappings:
             dynamic_templates:
@@ -4415,8 +4419,9 @@ elasticsearch:
     so-zeek:
       index_sorting: False
       index_template:
+        data_stream: {}
         index_patterns:
-          - so-zeek*
+          - logs-*-zeek_so
         template:
           mappings:
             dynamic_templates:

salt/elasticsearch/files/ingest-dynamic/common

@@ -57,16 +57,7 @@
     { "convert":         { "field": "log.id.uid", "type": "string",  "ignore_failure": true,  "ignore_missing": true  } },
     { "convert":         { "field": "agent.id", "type": "string",  "ignore_failure": true,  "ignore_missing": true  } },
     { "convert":         { "field": "event.severity", "type": "integer",  "ignore_failure": true,  "ignore_missing": true  } },
-    { "remove":          { "field": [ "message2", "type", "fields", "category", "module", "dataset" ], "ignore_missing": true, "ignore_failure": true } }, 
-    {
-        "date_index_name": {
-                "field": "@timestamp",
-                "index_name_prefix": "{{ _index }}-",
-                "date_rounding": "d",
-                "ignore_failure": true,
-                "index_name_format": "yyyy.MM.dd"
-        }
-    }
+    { "remove":          { "field": [ "message2", "type", "fields", "category", "module", "dataset" ], "ignore_missing": true, "ignore_failure": true } }
 {%- endraw %}
 {%- if HIGHLANDER %}
     ,

salt/elasticsearch/files/ingest/suricata.alert

@@ -1,6 +1,7 @@
 {
   "description" : "suricata.alert",
   "processors" : [
+    { "set":   { "field": "tags","value": "alert" }},
     { "rename":{ "field": "message2.alert",	"target_field": "rule",	"ignore_failure": true 	} },
     { "rename":{ "field": "rule.signature",     "target_field": "rule.name", "ignore_failure": true  } },
     { "rename":{ "field": "rule.ref",     "target_field": "rule.version", "ignore_failure": true  } },

salt/logstash/pipelines/config/so/9805_output_elastic_agent.conf.jinja

@@ -1,14 +1,28 @@
 output {
   if "elastic-agent" in [tags] and "import" not in [tags] {
-      elasticsearch {
-        hosts => "{{ GLOBALS.manager }}"
-        ecs_compatibility => v8
-        data_stream => true
-        user => "{{ ES_USER }}"
-        password => "{{ ES_PASS }}"
-        ssl => true
-        ssl_certificate_verification => false
+      if [metadata][pipeline] {
+        elasticsearch {
+          hosts => "{{ GLOBALS.manager }}"
+          ecs_compatibility => v8
+          data_stream => true
+          user => "{{ ES_USER }}"
+          password => "{{ ES_PASS }}"
+          pipeline => "%{[metadata][pipeline]}"
+          ssl => true
+          ssl_certificate_verification => false
+        }
       }
+      else {
+        elasticsearch {
+          hosts => "{{ GLOBALS.manager }}"
+          ecs_compatibility => v8
+          data_stream => true
+          user => "{{ ES_USER }}"
+          password => "{{ ES_PASS }}"
+          ssl => true
+          ssl_certificate_verification => false    
+        }
+     }
   }
 }
 

salt/top.sls

@@ -8,7 +8,7 @@
 {% set PLAYBOOK = salt['pillar.get']('manager:playbook', '0') %}
 {% set ELASTALERT = salt['pillar.get']('elastalert:enabled', True) %}
 {% set ELASTICSEARCH = salt['pillar.get']('elasticsearch:enabled', True) %}
-{% set FILEBEAT = salt['pillar.get']('filebeat:enabled', True) %}
+{% set FILEBEAT = salt['pillar.get']('filebeat:enabled', False) %}
 {% set KIBANA = salt['pillar.get']('kibana:enabled', True) %}
 {% set LOGSTASH = salt['pillar.get']('logstash:enabled', True) %}
 {% set REDIS = salt['pillar.get']('redis:enabled', True) %}
@@ -62,7 +62,9 @@ base:
     {%- if STRELKA %}
     - strelka
     {%- endif %}
+    {%- if FILEBEAT %}
     - filebeat
+    {%- endif %}
     - schedule
     - docker_clean
 
@@ -377,7 +379,9 @@ base:
     - firewall
     - schedule
     - docker_clean
+    {%- if FILEBEAT %}
     - filebeat
+    {%- endif %}
     - idh
 
   'J@workstation:gui:enabled:^[Tt][Rr][Uu][Ee]$ and ( G@saltversion:{{saltversion}} and G@os:CentOS )':