Merge pull request #15418 from Security-Onion-Solutions/reyesj2-patch-11

update heavynode's elastic-agent standalone policy

salt/elasticagent/files/elastic-agent.yml.jinja

@@ -3,7 +3,7 @@
 {%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %}
 
 id: aea1ba80-1065-11ee-a369-97538913b6a9
-revision: 1
+revision: 4
 outputs:
   default:
     type: elasticsearch
@@ -22,242 +22,133 @@ agent:
     metrics: false
   features: {}
 inputs:
-  - id: logfile-logs-fefef78c-422f-4cfa-8abf-4cd1b9428f62
+  - id: filestream-filestream-85820eb0-25ef-11f0-a18d-1b26f69b8310
-    name: import-evtx-logs
-    revision: 2
-    type: logfile
-    use_output: default
-    meta:
-      package:
-        name: log
-        version:
-    data_stream:
-      namespace: so
-    package_policy_id: fefef78c-422f-4cfa-8abf-4cd1b9428f62
-    streams:
-      - id: logfile-log.log-fefef78c-422f-4cfa-8abf-4cd1b9428f62
-        data_stream:
-          dataset: import
-        paths:
-          - /nsm/import/*/evtx/*.json
-        processors:
-          - dissect:
-              field: log.file.path
-              tokenizer: '/nsm/import/%{import.id}/evtx/%{import.file}'
-              target_prefix: ''
-          - decode_json_fields:
-              fields:
-                - message
-              target: ''
-          - drop_fields:
-              ignore_missing: true
-              fields:
-                - host
-          - add_fields:
-              fields:
-                dataset: system.security
-                type: logs
-                namespace: default
-              target: data_stream
-          - add_fields:
-              fields:
-                dataset: system.security
-                module: system
-                imported: true
-              target: event
-          - then:
-              - add_fields:
-                  fields:
-                    dataset: windows.sysmon_operational
-                  target: data_stream
-              - add_fields:
-                  fields:
-                    dataset: windows.sysmon_operational
-                    module: windows
-                    imported: true
-                  target: event
-            if:
-              equals:
-                winlog.channel: Microsoft-Windows-Sysmon/Operational
-          - then:
-              - add_fields:
-                  fields:
-                    dataset: system.application
-                  target: data_stream
-              - add_fields:
-                  fields:
-                    dataset: system.application
-                  target: event
-            if:
-              equals:
-                winlog.channel: Application
-          - then:
-              - add_fields:
-                  fields:
-                    dataset: system.system
-                  target: data_stream
-              - add_fields:
-                  fields:
-                    dataset: system.system
-                  target: event
-            if:
-              equals:
-                winlog.channel: System
-          - then:
-              - add_fields:
-                  fields:
-                    dataset: windows.powershell_operational
-                  target: data_stream
-              - add_fields:
-                  fields:
-                    dataset: windows.powershell_operational
-                    module: windows
-                  target: event
-            if:
-              equals:
-                winlog.channel: Microsoft-Windows-PowerShell/Operational
-        tags:
-          - import
-  - id: logfile-redis-fc98c947-7d17-4861-a318-7ad075f6d1b0
-    name: redis-logs
-    revision: 2
-    type: logfile
-    use_output: default
-    meta:
-      package:
-        name: redis
-        version:
-    data_stream:
-      namespace: default
-    package_policy_id: fc98c947-7d17-4861-a318-7ad075f6d1b0
-    streams:
-      - id: logfile-redis.log-fc98c947-7d17-4861-a318-7ad075f6d1b0
-        data_stream:
-          dataset: redis.log
-          type: logs
-        exclude_files:
-          - .gz$
-        paths:
-          - /opt/so/log/redis/redis.log
-        tags:
-          - redis-log
-        exclude_lines:
-          - '^\s+[\-`(''.|_]'
-  - id: logfile-logs-3b56803d-5ade-4c93-b25e-9b37182f66b8
     name: import-suricata-logs
-    revision: 2
+    revision: 3
-    type: logfile
+    type: filestream
     use_output: default
     meta:
       package:
-        name: log
+        name: filestream
         version:
     data_stream:
       namespace: so
-    package_policy_id: 3b56803d-5ade-4c93-b25e-9b37182f66b8
+    package_policy_id: 85820eb0-25ef-11f0-a18d-1b26f69b8310
     streams:
-      - id: logfile-log.log-3b56803d-5ade-4c93-b25e-9b37182f66b8
+      - id: filestream-filestream.generic-85820eb0-25ef-11f0-a18d-1b26f69b8310
         data_stream:
           dataset: import
-        pipeline: suricata.common
         paths:
           - /nsm/import/*/suricata/eve*.json
+        pipeline: suricata.common
+        prospector.scanner.recursive_glob: true
+        prospector.scanner.exclude_files:
+          - \.gz$
+        ignore_older: 72h
+        clean_inactive: -1
+        parsers: null
         processors:
           - add_fields:
+              target: event
               fields:
+                category: network
                 module: suricata
                 imported: true
-                category: network
-              target: event
           - dissect:
+              tokenizer: /nsm/import/%{import.id}/suricata/%{import.file}
               field: log.file.path
-              tokenizer: '/nsm/import/%{import.id}/suricata/%{import.file}'
               target_prefix: ''
-  - id: logfile-logs-c327e1a3-1ebe-449c-a8eb-f6f35032e69d
+        file_identity.native: null
-    name: soc-server-logs
+        prospector.scanner.fingerprint.enabled: false
-    revision: 2
+  - id: filestream-filestream-86b4e960-25ef-11f0-a18d-1b26f69b8310
-    type: logfile
+    name: import-zeek-logs
+    revision: 3
+    type: filestream
     use_output: default
     meta:
       package:
-        name: log
+        name: filestream
         version:
     data_stream:
       namespace: so
-    package_policy_id: c327e1a3-1ebe-449c-a8eb-f6f35032e69d
+    package_policy_id: 86b4e960-25ef-11f0-a18d-1b26f69b8310
     streams:
-      - id: logfile-log.log-c327e1a3-1ebe-449c-a8eb-f6f35032e69d
+      - id: filestream-filestream.generic-86b4e960-25ef-11f0-a18d-1b26f69b8310
         data_stream:
-          dataset: soc
+          dataset: import
-        pipeline: common
         paths:
-          - /opt/so/log/soc/sensoroni-server.log
+          - /nsm/import/*/zeek/logs/*.log
+        prospector.scanner.recursive_glob: true
+        prospector.scanner.exclude_files:
+          - >-
+            (broker|capture_loss|cluster|conn-summary|console|ecat_arp_info|known_certs|known_hosts|known_services|loaded_scripts|ntp|ocsp|packet_filter|reporter|stats|stderr|stdout).log$
+        clean_inactive: -1
+        parsers: null
         processors:
-          - decode_json_fields:
+          - dissect:
-              add_error_key: true
+              tokenizer: /nsm/import/%{import.id}/zeek/logs/%{import.file}
-              process_array: true
+              field: log.file.path
-              max_depth: 2
+              target_prefix: ''
-              fields:
+          - script:
-                - message
+              lang: javascript
-              target: soc
+              source: |
+                function process(event) {
+                  var pl = event.Get("import.file").slice(0,-4);
+                  event.Put("@metadata.pipeline", "zeek." + pl);
+                }
           - add_fields:
-              fields:
-                module: soc
-                dataset_temp: server
-                category: host
               target: event
-          - rename:
-              ignore_missing: true
               fields:
-                - from: soc.fields.sourceIp
+                category: network
-                  to: source.ip
+                module: zeek
-                - from: soc.fields.status
+                imported: true
-                  to: http.response.status_code
+          - add_tags:
-                - from: soc.fields.method
+              tags: ics
-                  to: http.request.method
+              when:
-                - from: soc.fields.path
+                regexp:
-                  to: url.path
+                  import.file: >-
-                - from: soc.message
+                    ^bacnet*|^bsap*|^cip*|^cotp*|^dnp3*|^ecat*|^enip*|^modbus*|^opcua*|^profinet*|^s7comm*
-                  to: event.action
+        file_identity.native: null
-                - from: soc.level
+        prospector.scanner.fingerprint.enabled: false
-                  to: log.level
+  - id: filestream-filestream-91741240-25ef-11f0-a18d-1b26f69b8310
-        tags:
-          - so-soc
-  - id: logfile-logs-906e0d4c-9ec3-4c6a-bef6-e347ec9fd073
     name: soc-sensoroni-logs
-    revision: 2
+    revision: 3
-    type: logfile
+    type: filestream
     use_output: default
     meta:
       package:
-        name: log
+        name: filestream
         version:
     data_stream:
       namespace: so
-    package_policy_id: 906e0d4c-9ec3-4c6a-bef6-e347ec9fd073
+    package_policy_id: 91741240-25ef-11f0-a18d-1b26f69b8310
     streams:
-      - id: logfile-log.log-906e0d4c-9ec3-4c6a-bef6-e347ec9fd073
+      - id: filestream-filestream.generic-91741240-25ef-11f0-a18d-1b26f69b8310
         data_stream:
           dataset: soc
-        pipeline: common
         paths:
           - /opt/so/log/sensoroni/sensoroni.log
+        pipeline: common
+        prospector.scanner.recursive_glob: true
+        prospector.scanner.exclude_files:
+          - \.gz$
+        clean_inactive: -1
+        parsers: null
         processors:
           - decode_json_fields:
-              add_error_key: true
-              process_array: true
-              max_depth: 2
               fields:
                 - message
               target: sensoroni
+              process_array: true
+              max_depth: 2
+              add_error_key: true
           - add_fields:
+              target: event
               fields:
+                category: host
                 module: soc
                 dataset_temp: sensoroni
-                category: host
-              target: event
           - rename:
-              ignore_missing: true
               fields:
                 - from: sensoroni.fields.sourceIp
                   to: source.ip
@@ -271,141 +162,100 @@ inputs:
                   to: event.action
                 - from: sensoroni.level
                   to: log.level
-  - id: logfile-logs-df0d7f2c-221f-433b-b18b-d1cf83250515
+              ignore_missing: true
-    name: soc-salt-relay-logs
+        file_identity.native: null
-    revision: 2
+        prospector.scanner.fingerprint.enabled: false
-    type: logfile
+  - id: filestream-filestream-976e3900-25ef-11f0-a18d-1b26f69b8310
-    use_output: default
-    meta:
-      package:
-        name: log
-        version:
-    data_stream:
-      namespace: so
-    package_policy_id: df0d7f2c-221f-433b-b18b-d1cf83250515
-    streams:
-      - id: logfile-log.log-df0d7f2c-221f-433b-b18b-d1cf83250515
-        data_stream:
-          dataset: soc
-        pipeline: common
-        paths:
-          - /opt/so/log/soc/salt-relay.log
-        processors:
-          - dissect:
-              field: message
-              tokenizer: '%{soc.ts} | %{event.action}'
-              target_prefix: ''
-          - add_fields:
-              fields:
-                module: soc
-                dataset_temp: salt_relay
-                category: host
-              target: event
-        tags:
-          - so-soc
-  - id: logfile-logs-74bd2366-fe52-493c-bddc-843a017fc4d0
-    name: soc-auth-sync-logs
-    revision: 2
-    type: logfile
-    use_output: default
-    meta:
-      package:
-        name: log
-        version:
-    data_stream:
-      namespace: so
-    package_policy_id: 74bd2366-fe52-493c-bddc-843a017fc4d0
-    streams:
-      - id: logfile-log.log-74bd2366-fe52-493c-bddc-843a017fc4d0
-        data_stream:
-          dataset: soc
-        pipeline: common
-        paths:
-          - /opt/so/log/soc/sync.log
-        processors:
-          - dissect:
-              field: message
-              tokenizer: '%{event.action}'
-              target_prefix: ''
-          - add_fields:
-              fields:
-                module: soc
-                dataset_temp: auth_sync
-                category: host
-              target: event
-        tags:
-          - so-soc
-  - id: logfile-logs-d151d9bf-ff2a-4529-9520-c99244bc0253
     name: suricata-logs
-    revision: 2
+    revision: 3
-    type: logfile
+    type: filestream
     use_output: default
     meta:
       package:
-        name: log
+        name: filestream
         version:
     data_stream:
       namespace: so
-    package_policy_id: d151d9bf-ff2a-4529-9520-c99244bc0253
+    package_policy_id: 976e3900-25ef-11f0-a18d-1b26f69b8310
     streams:
-      - id: logfile-log.log-d151d9bf-ff2a-4529-9520-c99244bc0253
+      - id: filestream-filestream.generic-976e3900-25ef-11f0-a18d-1b26f69b8310
         data_stream:
           dataset: suricata
-        pipeline: suricata.common
         paths:
           - /nsm/suricata/eve*.json
+        pipeline: suricata.common
+        prospector.scanner.recursive_glob: true
+        prospector.scanner.exclude_files:
+          - \.gz$
+        clean_inactive: -1
+        parsers: null
         processors:
           - add_fields:
-              fields:
-                module: suricata
-                category: network
               target: event
-  - id: logfile-logs-31f94d05-ae75-40ee-b9c5-0e0356eff327
+              fields:
+                category: network
+                module: suricata
+        file_identity.native: null
+        prospector.scanner.fingerprint.enabled: false
+  - id: filestream-filestream-95091fe0-25ef-11f0-a18d-1b26f69b8310
     name: strelka-logs
-    revision: 2
+    revision: 3
-    type: logfile
+    type: filestream
     use_output: default
     meta:
       package:
-        name: log
+        name: filestream
         version:
     data_stream:
       namespace: so
-    package_policy_id: 31f94d05-ae75-40ee-b9c5-0e0356eff327
+    package_policy_id: 95091fe0-25ef-11f0-a18d-1b26f69b8310
     streams:
-      - id: logfile-log.log-31f94d05-ae75-40ee-b9c5-0e0356eff327
+      - id: filestream-filestream.generic-95091fe0-25ef-11f0-a18d-1b26f69b8310
         data_stream:
           dataset: strelka
-        pipeline: strelka.file
         paths:
           - /nsm/strelka/log/strelka.log
+        pipeline: strelka.file
+        prospector.scanner.recursive_glob: true
+        prospector.scanner.exclude_files:
+          - \.gz$
+        clean_inactive: -1
+        parsers: null
         processors:
           - add_fields:
-              fields:
-                module: strelka
-                category: file
               target: event
-  - id: logfile-logs-6197fe84-9b58-4d9b-8464-3d517f28808d
+              fields:
+                category: file
+                module: strelka
+        file_identity.native: null
+        prospector.scanner.fingerprint.enabled: false
+  - id: filestream-filestream-9f309ca0-25ef-11f0-a18d-1b26f69b8310
     name: zeek-logs
-    revision: 1
+    revision: 2
-    type: logfile
+    type: filestream
     use_output: default
     meta:
       package:
-        name: log
+        name: filestream
-        version: 
+        version:
     data_stream:
       namespace: so
-    package_policy_id: 6197fe84-9b58-4d9b-8464-3d517f28808d
+    package_policy_id: 9f309ca0-25ef-11f0-a18d-1b26f69b8310
     streams:
-      - id: logfile-log.log-6197fe84-9b58-4d9b-8464-3d517f28808d
+      - id: filestream-filestream.generic-9f309ca0-25ef-11f0-a18d-1b26f69b8310
         data_stream:
           dataset: zeek
         paths:
           - /nsm/zeek/logs/current/*.log
+        prospector.scanner.recursive_glob: true
+        prospector.scanner.exclude_files:
+          - >-
+            (broker|capture_loss|cluster|conn-summary|console|ecat_arp_info|known_certs|known_hosts|known_services|loaded_scripts|ntp|ocsp|packet_filter|reporter|stats|stderr|stdout).log$
+        clean_inactive: -1
+        parsers: null
         processors:
           - dissect:
-              tokenizer: '/nsm/zeek/logs/current/%{pipeline}.log'
+              tokenizer: /nsm/zeek/logs/current/%{pipeline}.log
               field: log.file.path
               trim_chars: .log
               target_prefix: ''
@@ -427,18 +277,17 @@ inputs:
                 regexp:
                   pipeline: >-
                     ^bacnet*|^bsap*|^cip*|^cotp*|^dnp3*|^ecat*|^enip*|^modbus*|^opcua*|^profinet*|^s7comm*
-        exclude_files:
+        file_identity.native: null
-          - >-
+        prospector.scanner.fingerprint.enabled: false
-            broker|capture_loss|cluster|ecat_arp_info|known_hosts|known_services|loaded_scripts|ntp|ocsp|packet_filter|reporter|stats|stderr|stdout.log$
   - id: udp-udp-35051de0-46a5-11ee-8d5d-9f98c8182f60
     name: syslog-udp-514
-    revision: 3
+    revision: 4
     type: udp
     use_output: default
     meta:
       package:
         name: udp
-        version: 1.10.0
+        version:
     data_stream:
       namespace: so
     package_policy_id: 35051de0-46a5-11ee-8d5d-9f98c8182f60
@@ -458,13 +307,13 @@ inputs:
           - syslog
   - id: tcp-tcp-33d37bb0-46a5-11ee-8d5d-9f98c8182f60
     name: syslog-tcp-514
-    revision: 3
+    revision: 4
     type: tcp
     use_output: default
     meta:
       package:
         name: tcp
-        version: 1.10.0
+        version:
     data_stream:
       namespace: so
     package_policy_id: 33d37bb0-46a5-11ee-8d5d-9f98c8182f60

salt/elasticfleet/files/integrations/grid-nodes_heavy/elasticsearch-grid-nodes.json

@@ -0,0 +1,107 @@
+{
+  "package": {
+    "name": "elasticsearch",
+    "version": ""
+  },
+  "name": "elasticsearch-grid-nodes_heavy",
+  "namespace": "default",
+  "description": "Elasticsearch Logs",
+  "policy_id": "so-grid-nodes_heavy",
+  "inputs": {
+    "elasticsearch-logfile": {
+      "enabled": true,
+      "streams": {
+        "elasticsearch.audit": {
+          "enabled": false,
+          "vars": {
+            "paths": [
+              "/var/log/elasticsearch/*_audit.json"
+            ]
+          }
+        },
+        "elasticsearch.deprecation": {
+          "enabled": false,
+          "vars": {
+            "paths": [
+              "/var/log/elasticsearch/*_deprecation.json"
+            ]
+          }
+        },
+        "elasticsearch.gc": {
+          "enabled": false,
+          "vars": {
+            "paths": [
+              "/var/log/elasticsearch/gc.log.[0-9]*",
+              "/var/log/elasticsearch/gc.log"
+            ]
+          }
+        },
+        "elasticsearch.server": {
+          "enabled": true,
+          "vars": {
+            "paths": [
+              "/opt/so/log/elasticsearch/*.json"
+            ]
+          }
+        },
+        "elasticsearch.slowlog": {
+          "enabled": false,
+          "vars": {
+            "paths": [
+              "/var/log/elasticsearch/*_index_search_slowlog.json",
+              "/var/log/elasticsearch/*_index_indexing_slowlog.json"
+            ]
+          }
+        }
+      }
+    },
+    "elasticsearch-elasticsearch/metrics": {
+      "enabled": false,
+      "vars": {
+        "hosts": [
+          "http://localhost:9200"
+        ],
+        "scope": "node"
+      },
+      "streams": {
+        "elasticsearch.stack_monitoring.ccr": {
+          "enabled": false
+        },
+        "elasticsearch.stack_monitoring.cluster_stats": {
+          "enabled": false
+        },
+        "elasticsearch.stack_monitoring.enrich": {
+          "enabled": false
+        },
+        "elasticsearch.stack_monitoring.index": {
+          "enabled": false
+        },
+        "elasticsearch.stack_monitoring.index_recovery": {
+          "enabled": false,
+          "vars": {
+            "active.only": true
+          }
+        },
+        "elasticsearch.stack_monitoring.index_summary": {
+          "enabled": false
+        },
+        "elasticsearch.stack_monitoring.ml_job": {
+          "enabled": false
+        },
+        "elasticsearch.stack_monitoring.node": {
+          "enabled": false
+        },
+        "elasticsearch.stack_monitoring.node_stats": {
+          "enabled": false
+        },
+        "elasticsearch.stack_monitoring.pending_tasks": {
+          "enabled": false
+        },
+        "elasticsearch.stack_monitoring.shard": {
+          "enabled": false
+        }
+      }
+    }
+  },
+  "force": true
+}

salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load

@@ -17,9 +17,9 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then
 
   # Third, configure Elastic Defend Integration seperately
   /usr/sbin/so-elastic-fleet-integration-policy-elastic-defend
+
   # Initial Endpoints
-  for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/endpoints-initial/*.json
+  for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/endpoints-initial/*.json; do
-  do
     printf "\n\nInitial Endpoints Policy - Loading $INTEGRATION\n"
     elastic_fleet_integration_check "endpoints-initial" "$INTEGRATION"
     if [ -n "$INTEGRATION_ID" ]; then
@@ -40,8 +40,7 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then
   done
 
   # Grid Nodes - General
-  for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/grid-nodes_general/*.json
+  for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/grid-nodes_general/*.json; do
-  do
     printf "\n\nGrid Nodes Policy_General - Loading $INTEGRATION\n"
     elastic_fleet_integration_check "so-grid-nodes_general" "$INTEGRATION"
     if [ -n "$INTEGRATION_ID" ]; then
@@ -60,13 +59,9 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then
       fi
     fi
   done
-  if [[ "$RETURN_CODE" != "1" ]]; then
-    touch /opt/so/state/eaintegrations.txt
-  fi
 
   # Grid Nodes - Heavy
-  for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/grid-nodes_heavy/*.json
+  for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/grid-nodes_heavy/*.json; do
-  do
     printf "\n\nGrid Nodes Policy_Heavy - Loading $INTEGRATION\n"
     elastic_fleet_integration_check "so-grid-nodes_heavy" "$INTEGRATION"
     if [ -n "$INTEGRATION_ID" ]; then
@@ -78,22 +73,16 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then
       fi
     else
       printf "\n\nIntegration does not exist - Creating integration\n"
-      if [ "$NAME" != "elasticsearch-logs" ]; then
+      if ! elastic_fleet_integration_create "@$INTEGRATION"; then
-        if ! elastic_fleet_integration_create "@$INTEGRATION"; then
+        echo -e "\nFailed to create integration for ${INTEGRATION##*/}"
-          echo -e "\nFailed to create integration for ${INTEGRATION##*/}"
+        RETURN_CODE=1
-          RETURN_CODE=1
+        continue
-          continue
-        fi
       fi
     fi
   done
-  if [[ "$RETURN_CODE" != "1" ]]; then
-    touch /opt/so/state/eaintegrations.txt
-  fi
 
   # Fleet Server - Optional integrations
-  for INTEGRATION in /opt/so/conf/elastic-fleet/integrations-optional/FleetServer*/*.json
+  for INTEGRATION in /opt/so/conf/elastic-fleet/integrations-optional/FleetServer*/*.json; do
-  do
     if ! [ "$INTEGRATION" == "/opt/so/conf/elastic-fleet/integrations-optional/FleetServer*/*.json" ]; then
       FLEET_POLICY=`echo "$INTEGRATION"| cut -d'/' -f7`
       printf "\n\nFleet Server Policy - Loading $INTEGRATION\n"
@@ -117,6 +106,8 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then
       fi
     fi
   done
+
+  # Only create the state file if all policies were created/updated successfully
   if [[ "$RETURN_CODE" != "1" ]]; then
     touch /opt/so/state/eaintegrations.txt
   fi

salt/elasticsearch/defaults.yaml

@@ -691,7 +691,6 @@ elasticsearch:
                 match_mapping_type: string
           settings:
             index:
-              final_pipeline: .fleet_final_pipeline-1
               lifecycle:
                 name: so-import-logs
               mapping:

salt/elasticsearch/files/ingest/global@custom

@@ -1,31 +1,212 @@
 {
-  "version": 3,
+    "version": 3,
-  "_meta": {
+    "_meta": {
-    "managed_by": "securityonion",
+        "managed_by": "securityonion",
-    "managed": true
+        "managed": true
-  },
+    },
-  "description": "Custom pipeline for processing all incoming Fleet Agent documents. \n",
+    "description": "Custom pipeline for processing all incoming Fleet Agent documents. \n",
-  "processors": [
+    "processors": [
-    { "set":        { "ignore_failure": true,  "field": "event.module", "value": "elastic_agent" } },
+        {
-    { "split":      { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "separator": "\\.", "target_field": "module_temp"  } },
+            "set": {
-    { "split":      { "if": "ctx.data_stream?.dataset != null && ctx.data_stream?.dataset.contains('.')", "field":"data_stream.dataset", "separator":"\\.", "target_field":"datastream_dataset_temp", "ignore_missing":true } },
+                "ignore_failure": true,
-    { "set":        { "if": "ctx.module_temp != null", "override": true, "field": "event.module", "value": "{{module_temp.0}}" } },
+                "field": "event.module",
-    { "set":        { "if": "ctx.datastream_dataset_temp != null && ctx.datastream_dataset_temp[0] == 'network_traffic'", "field":"event.module", "value":"{{ datastream_dataset_temp.0 }}", "ignore_failure":true, "ignore_empty_value":true, "description":"Fix EA network packet capture" } },
+                "value": "elastic_agent"
-    { "gsub":       { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "pattern": "^[^.]*.", "replacement": "", "target_field": "dataset_tag_temp" } },
+            }
-    { "append":     { "if": "ctx.dataset_tag_temp != null", "field": "tags", "value": "{{dataset_tag_temp}}", "allow_duplicates": false  } },
+        },
-    { "set":        { "if": "ctx.network?.direction == 'egress'", "override": true, "field": "network.initiated", "value": "true" } },
+        {
-    { "set":        { "if": "ctx.network?.direction == 'ingress'", "override": true, "field": "network.initiated", "value": "false" } },
+            "split": {
-    { "set":        { "if": "ctx.network?.type == 'ipv4'", "override": true, "field": "destination.ipv6", "value": "false" } },
+                "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')",
-    { "set":        { "if": "ctx.network?.type == 'ipv6'", "override": true, "field": "destination.ipv6", "value": "true" } },
+                "field": "event.dataset",
-    { "set":        { "if": "ctx.tags != null && ctx.tags.contains('import')", "override": true, "field": "data_stream.dataset", "value": "import" } },
+                "separator": "\\.",
-    { "set":        { "if": "ctx.tags != null && ctx.tags.contains('import')", "override": true, "field": "data_stream.namespace", "value": "so" } },
+                "target_field": "module_temp"
-    { "community_id":{ "if": "ctx.event?.dataset == 'endpoint.events.network'", "ignore_failure":true }  },
+            }
-    { "set":        { "if": "ctx.event?.module == 'fim'", "override": true, "field": "event.module", "value": "file_integrity" } },
+        },
-    { "rename":       { "if": "ctx.winlog?.provider_name == 'Microsoft-Windows-Windows Defender'", "ignore_missing": true, "field": "winlog.event_data.Threat Name", "target_field": "winlog.event_data.threat_name" } },
+        {
-    { "set":        { "if": "ctx?.metadata?.kafka != null" , "field": "kafka.id", "value": "{{metadata.kafka.partition}}{{metadata.kafka.offset}}{{metadata.kafka.timestamp}}", "ignore_failure": true } },
+            "split": {
-    { "set":        { "if": "ctx.event?.dataset != null && ctx.event?.dataset == 'elasticsearch.server'", "field": "event.module", "value":"elasticsearch" }},
+                "if": "ctx.data_stream?.dataset != null && ctx.data_stream?.dataset.contains('.')",
-    {"append":      {"field":"related.ip","value":["{{source.ip}}","{{destination.ip}}"],"allow_duplicates":false,"if":"ctx?.event?.dataset == 'endpoint.events.network' && ctx?.source?.ip != null","ignore_failure":true}},
+                "field": "data_stream.dataset",
-    {"foreach":     {"field":"host.ip","processor":{"append":{"field":"related.ip","value":"{{_ingest._value}}","allow_duplicates":false}},"if":"ctx?.event?.module == 'endpoint' && ctx?.host?.ip != null","ignore_missing":true, "description":"Extract IPs from Elastic Agent events (host.ip) and adds them to related.ip"}},
+                "separator": "\\.",
-    { "remove":     { "field": [ "message2", "type", "fields", "category", "module", "dataset", "event.dataset_temp", "dataset_tag_temp", "module_temp", "datastream_dataset_temp" ], "ignore_missing": true, "ignore_failure": true } }
+                "target_field": "datastream_dataset_temp",
-  ]
+                "ignore_missing": true
-}
+            }
+        },
+        {
+            "set": {
+                "if": "ctx.module_temp != null",
+                "override": true,
+                "field": "event.module",
+                "value": "{{module_temp.0}}"
+            }
+        },
+        {
+            "set": {
+                "if": "ctx.datastream_dataset_temp != null && ctx.datastream_dataset_temp[0] == 'network_traffic'",
+                "field": "event.module",
+                "value": "{{ datastream_dataset_temp.0 }}",
+                "ignore_failure": true,
+                "ignore_empty_value": true,
+                "description": "Fix EA network packet capture"
+            }
+        },
+        {
+            "gsub": {
+                "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')",
+                "field": "event.dataset",
+                "pattern": "^[^.]*.",
+                "replacement": "",
+                "target_field": "dataset_tag_temp"
+            }
+        },
+        {
+            "append": {
+                "if": "ctx.dataset_tag_temp != null",
+                "field": "tags",
+                "value": "{{dataset_tag_temp}}",
+                "allow_duplicates": false
+            }
+        },
+        {
+            "set": {
+                "if": "ctx.network?.direction == 'egress'",
+                "override": true,
+                "field": "network.initiated",
+                "value": "true"
+            }
+        },
+        {
+            "set": {
+                "if": "ctx.network?.direction == 'ingress'",
+                "override": true,
+                "field": "network.initiated",
+                "value": "false"
+            }
+        },
+        {
+            "set": {
+                "if": "ctx.network?.type == 'ipv4'",
+                "override": true,
+                "field": "destination.ipv6",
+                "value": "false"
+            }
+        },
+        {
+            "set": {
+                "if": "ctx.network?.type == 'ipv6'",
+                "override": true,
+                "field": "destination.ipv6",
+                "value": "true"
+            }
+        },
+        {
+            "set": {
+                "if": "ctx.tags != null && ctx.tags.contains('import')",
+                "override": true,
+                "field": "data_stream.dataset",
+                "value": "import"
+            }
+        },
+        {
+            "set": {
+                "if": "ctx.tags != null && ctx.tags.contains('import')",
+                "override": true,
+                "field": "data_stream.namespace",
+                "value": "so"
+            }
+        },
+        {
+            "community_id": {
+                "if": "ctx.event?.dataset == 'endpoint.events.network'",
+                "ignore_failure": true
+            }
+        },
+        {
+            "set": {
+                "if": "ctx.event?.module == 'fim'",
+                "override": true,
+                "field": "event.module",
+                "value": "file_integrity"
+            }
+        },
+        {
+            "rename": {
+                "if": "ctx.winlog?.provider_name == 'Microsoft-Windows-Windows Defender'",
+                "ignore_missing": true,
+                "field": "winlog.event_data.Threat Name",
+                "target_field": "winlog.event_data.threat_name"
+            }
+        },
+        {
+            "set": {
+                "if": "ctx?.metadata?.kafka != null",
+                "field": "kafka.id",
+                "value": "{{metadata.kafka.partition}}{{metadata.kafka.offset}}{{metadata.kafka.timestamp}}",
+                "ignore_failure": true
+            }
+        },
+        {
+            "set": {
+                "if": "ctx.event?.dataset != null && ctx.event?.dataset == 'elasticsearch.server'",
+                "field": "event.module",
+                "value": "elasticsearch"
+            }
+        },
+        {
+            "append": {
+                "field": "related.ip",
+                "value": [
+                    "{{source.ip}}",
+                    "{{destination.ip}}"
+                ],
+                "allow_duplicates": false,
+                "if": "ctx?.event?.dataset == 'endpoint.events.network' && ctx?.source?.ip != null",
+                "ignore_failure": true
+            }
+        },
+        {
+            "foreach": {
+                "field": "host.ip",
+                "processor": {
+                    "append": {
+                        "field": "related.ip",
+                        "value": "{{_ingest._value}}",
+                        "allow_duplicates": false
+                    }
+                },
+                "if": "ctx?.event?.module == 'endpoint' && ctx?.host?.ip != null",
+                "ignore_missing": true,
+                "description": "Extract IPs from Elastic Agent events (host.ip) and adds them to related.ip"
+            }
+        },
+        {
+            "pipeline": {
+                "name": ".fleet_final_pipeline-1",
+                "ignore_missing_pipeline": true
+            }
+        },
+        {
+            "remove": {
+                "field": "event.agent_id_status",
+                "ignore_missing": true,
+                "if": "ctx?.event?.agent_id_status == 'auth_metadata_missing'"
+            }
+        },
+        {
+            "remove": {
+                "field": [
+                    "message2",
+                    "type",
+                    "fields",
+                    "category",
+                    "module",
+                    "dataset",
+                    "event.dataset_temp",
+                    "dataset_tag_temp",
+                    "module_temp",
+                    "datastream_dataset_temp"
+                ],
+                "ignore_missing": true,
+                "ignore_failure": true
+            }
+        }
+    ]
+}

salt/elasticsearch/templates/component/elastic-agent/so-fleet_agent_id_verification-1.json

@@ -2,7 +2,7 @@
   "template": {
     "settings": {
       "index": {
-        "final_pipeline": ".fleet_final_pipeline-1"
+        "final_pipeline": "global@custom"
       }
     },
     "mappings": {

salt/manager/tools/sbin/so-minion

@@ -839,7 +839,6 @@ function createHEAVYNODE() {
 	add_elastic_agent_to_minion || return 1
 	add_sensor_to_minion || return 1
 	add_strelka_to_minion || return 1
-	add_redis_to_minion || return 1
 	add_telegraf_to_minion || return 1
 }
 

salt/manager/tools/sbin/soup

@@ -353,6 +353,22 @@ disable_logstash_heavynodes() {
   done
 }
 
+disable_redis_heavynodes() {
+  local c=0
+  printf "\nChecking for heavynodes and disabling Redis if they exist\n"
+  for file in /opt/so/saltstack/local/pillar/minions/*.sls; do
+    if [[ "$file" =~ "_heavynode.sls" && ! "$file" =~ "/opt/so/saltstack/local/pillar/minions/adv_" ]]; then
+      c=1
+      echo "Disabling Redis for: $file"
+      so-yaml.py replace "$file" redis.enabled False
+    fi
+  done
+
+  if [[ "$c" != 0 ]]; then
+    FINAL_MESSAGE_QUEUE+=("Redis has been disabled on all heavynodes.")
+  fi
+}
+
 enable_highstate() {
     echo "Enabling highstate."
     salt-call state.enable highstate -l info --local
@@ -674,6 +690,8 @@ post_to_2.4.210() {
 
   rollover_index "logs-kratos-so"
 
+  disable_redis_heavynodes
+
   echo "Regenerating Elastic Agent Installers"
   /sbin/so-elastic-agent-gen-installers
 

salt/telegraf/etc/telegraf.conf

@@ -7,6 +7,7 @@
 {%- set UNIQUEID = salt['pillar.get']('sensor:uniqueid', '') %}
 {%- set ZEEK_ENABLED = salt['pillar.get']('zeek:enabled', True) %}
 {%- set MDENGINE = GLOBALS.md_engine %}
+{%- set LOGSTASH_ENABLED = salt['pillar.get']('logstash:enabled', False) %}
 # Global tags can be specified here in key="value" format.
 [global_tags]
   role = "{{ GLOBALS.role.split('-') | last }}"
@@ -241,12 +242,8 @@
 #   ## Use TLS but skip chain & host verification
 #   # insecure_skip_verify = false
 
-{%- set logstash_metrics_roles = ['so-searchnode','so-standalone','so-managersearch','so-heavynode'] %}
+{#- Fleet nodes do not have pillar access to logstash credentials #}
-{%- if GLOBALS.pipeline != "KAFKA" %}
+{%- if LOGSTASH_ENABLED and grains.role != 'so-fleet' %}
-{%-   set logstash_metrics_roles = logstash_metrics_roles + ['so-manager', 'so-receiver'] %}
-{%- endif %}
-
-{%- if grains.role in logstash_metrics_roles %}
 [[inputs.logstash]]
   url = "http://localhost:9600"
   collect = ["pipelines"]