update redis log file path

salt/common/tools/sbin/so-log-check

@@ -163,6 +163,7 @@ if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|updating component template"  # false positive (elasticsearch index or template names contain 'error') 
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|upgrading component template"  # false positive (elasticsearch index or template names contain 'error')
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|upgrading composable template" # false positive (elasticsearch composable template names contain 'error')
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Error while parsing document for index \[.ds-logs-kratos-so-.*object mapping for \[file\]" # false positive (mapping error occuring BEFORE kratos index has rolled over in 2.4.210)
 fi
 
 if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then

salt/elasticagent/files/elastic-agent.yml.jinja

@@ -3,7 +3,7 @@
 {%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %}
 
 id: aea1ba80-1065-11ee-a369-97538913b6a9
-revision: 4
+revision: 1
 outputs:
   default:
     type: elasticsearch
@@ -22,133 +22,242 @@ agent:
     metrics: false
   features: {}
 inputs:
-  - id: filestream-filestream-85820eb0-25ef-11f0-a18d-1b26f69b8310
-    name: import-suricata-logs
-    revision: 3
-    type: filestream
+  - id: logfile-logs-fefef78c-422f-4cfa-8abf-4cd1b9428f62
+    name: import-evtx-logs
+    revision: 2
+    type: logfile
     use_output: default
     meta:
       package:
-        name: filestream
+        name: log
         version:
     data_stream:
       namespace: so
-    package_policy_id: 85820eb0-25ef-11f0-a18d-1b26f69b8310
+    package_policy_id: fefef78c-422f-4cfa-8abf-4cd1b9428f62
     streams:
-      - id: filestream-filestream.generic-85820eb0-25ef-11f0-a18d-1b26f69b8310
+      - id: logfile-log.log-fefef78c-422f-4cfa-8abf-4cd1b9428f62
         data_stream:
           dataset: import
         paths:
-          - /nsm/import/*/suricata/eve*.json
-        pipeline: suricata.common
-        prospector.scanner.recursive_glob: true
-        prospector.scanner.exclude_files:
-          - \.gz$
-        ignore_older: 72h
-        clean_inactive: -1
-        parsers: null
-        processors:
-          - add_fields:
-              target: event
-              fields:
-                category: network
-                module: suricata
-                imported: true
-          - dissect:
-              tokenizer: /nsm/import/%{import.id}/suricata/%{import.file}
-              field: log.file.path
-              target_prefix: ''
-        file_identity.native: null
-        prospector.scanner.fingerprint.enabled: false
-  - id: filestream-filestream-86b4e960-25ef-11f0-a18d-1b26f69b8310
-    name: import-zeek-logs
-    revision: 3
-    type: filestream
-    use_output: default
-    meta:
-      package:
-        name: filestream
-        version:
-    data_stream:
-      namespace: so
-    package_policy_id: 86b4e960-25ef-11f0-a18d-1b26f69b8310
-    streams:
-      - id: filestream-filestream.generic-86b4e960-25ef-11f0-a18d-1b26f69b8310
-        data_stream:
-          dataset: import
-        paths:
-          - /nsm/import/*/zeek/logs/*.log
-        prospector.scanner.recursive_glob: true
-        prospector.scanner.exclude_files:
-          - >-
-            (broker|capture_loss|cluster|conn-summary|console|ecat_arp_info|known_certs|known_hosts|known_services|loaded_scripts|ntp|ocsp|packet_filter|reporter|stats|stderr|stdout).log$
-        clean_inactive: -1
-        parsers: null
+          - /nsm/import/*/evtx/*.json
         processors:
           - dissect:
-              tokenizer: /nsm/import/%{import.id}/zeek/logs/%{import.file}
               field: log.file.path
+              tokenizer: '/nsm/import/%{import.id}/evtx/%{import.file}'
               target_prefix: ''
-          - script:
-              lang: javascript
-              source: |
-                function process(event) {
-                  var pl = event.Get("import.file").slice(0,-4);
-                  event.Put("@metadata.pipeline", "zeek." + pl);
-                }
-          - add_fields:
-              target: event
-              fields:
-                category: network
-                module: zeek
-                imported: true
-          - add_tags:
-              tags: ics
-              when:
-                regexp:
-                  import.file: >-
-                    ^bacnet*|^bsap*|^cip*|^cotp*|^dnp3*|^ecat*|^enip*|^modbus*|^opcua*|^profinet*|^s7comm*
-        file_identity.native: null
-        prospector.scanner.fingerprint.enabled: false
-  - id: filestream-filestream-91741240-25ef-11f0-a18d-1b26f69b8310
-    name: soc-sensoroni-logs
-    revision: 3
-    type: filestream
-    use_output: default
-    meta:
-      package:
-        name: filestream
-        version:
-    data_stream:
-      namespace: so
-    package_policy_id: 91741240-25ef-11f0-a18d-1b26f69b8310
-    streams:
-      - id: filestream-filestream.generic-91741240-25ef-11f0-a18d-1b26f69b8310
-        data_stream:
-          dataset: soc
-        paths:
-          - /opt/so/log/sensoroni/sensoroni.log
-        pipeline: common
-        prospector.scanner.recursive_glob: true
-        prospector.scanner.exclude_files:
-          - \.gz$
-        clean_inactive: -1
-        parsers: null
-        processors:
           - decode_json_fields:
               fields:
                 - message
-              target: sensoroni
+              target: ''
+          - drop_fields:
+              ignore_missing: true
+              fields:
+                - host
+          - add_fields:
+              fields:
+                dataset: system.security
+                type: logs
+                namespace: default
+              target: data_stream
+          - add_fields:
+              fields:
+                dataset: system.security
+                module: system
+                imported: true
+              target: event
+          - then:
+              - add_fields:
+                  fields:
+                    dataset: windows.sysmon_operational
+                  target: data_stream
+              - add_fields:
+                  fields:
+                    dataset: windows.sysmon_operational
+                    module: windows
+                    imported: true
+                  target: event
+            if:
+              equals:
+                winlog.channel: Microsoft-Windows-Sysmon/Operational
+          - then:
+              - add_fields:
+                  fields:
+                    dataset: system.application
+                  target: data_stream
+              - add_fields:
+                  fields:
+                    dataset: system.application
+                  target: event
+            if:
+              equals:
+                winlog.channel: Application
+          - then:
+              - add_fields:
+                  fields:
+                    dataset: system.system
+                  target: data_stream
+              - add_fields:
+                  fields:
+                    dataset: system.system
+                  target: event
+            if:
+              equals:
+                winlog.channel: System
+          - then:
+              - add_fields:
+                  fields:
+                    dataset: windows.powershell_operational
+                  target: data_stream
+              - add_fields:
+                  fields:
+                    dataset: windows.powershell_operational
+                    module: windows
+                  target: event
+            if:
+              equals:
+                winlog.channel: Microsoft-Windows-PowerShell/Operational
+        tags:
+          - import
+  - id: logfile-redis-fc98c947-7d17-4861-a318-7ad075f6d1b0
+    name: redis-logs
+    revision: 2
+    type: logfile
+    use_output: default
+    meta:
+      package:
+        name: redis
+        version:
+    data_stream:
+      namespace: default
+    package_policy_id: fc98c947-7d17-4861-a318-7ad075f6d1b0
+    streams:
+      - id: logfile-redis.log-fc98c947-7d17-4861-a318-7ad075f6d1b0
+        data_stream:
+          dataset: redis.log
+          type: logs
+        exclude_files:
+          - .gz$
+        paths:
+          - /opt/so/log/redis/redis.log
+        tags:
+          - redis-log
+        exclude_lines:
+          - '^\s+[\-`(''.|_]'
+  - id: logfile-logs-3b56803d-5ade-4c93-b25e-9b37182f66b8
+    name: import-suricata-logs
+    revision: 2
+    type: logfile
+    use_output: default
+    meta:
+      package:
+        name: log
+        version:
+    data_stream:
+      namespace: so
+    package_policy_id: 3b56803d-5ade-4c93-b25e-9b37182f66b8
+    streams:
+      - id: logfile-log.log-3b56803d-5ade-4c93-b25e-9b37182f66b8
+        data_stream:
+          dataset: import
+        pipeline: suricata.common
+        paths:
+          - /nsm/import/*/suricata/eve*.json
+        processors:
+          - add_fields:
+              fields:
+                module: suricata
+                imported: true
+                category: network
+              target: event
+          - dissect:
+              field: log.file.path
+              tokenizer: '/nsm/import/%{import.id}/suricata/%{import.file}'
+              target_prefix: ''
+  - id: logfile-logs-c327e1a3-1ebe-449c-a8eb-f6f35032e69d
+    name: soc-server-logs
+    revision: 2
+    type: logfile
+    use_output: default
+    meta:
+      package:
+        name: log
+        version:
+    data_stream:
+      namespace: so
+    package_policy_id: c327e1a3-1ebe-449c-a8eb-f6f35032e69d
+    streams:
+      - id: logfile-log.log-c327e1a3-1ebe-449c-a8eb-f6f35032e69d
+        data_stream:
+          dataset: soc
+        pipeline: common
+        paths:
+          - /opt/so/log/soc/sensoroni-server.log
+        processors:
+          - decode_json_fields:
+              add_error_key: true
               process_array: true
               max_depth: 2
-              add_error_key: true
-          - add_fields:
-              target: event
               fields:
+                - message
+              target: soc
+          - add_fields:
+              fields:
+                module: soc
+                dataset_temp: server
                 category: host
+              target: event
+          - rename:
+              ignore_missing: true
+              fields:
+                - from: soc.fields.sourceIp
+                  to: source.ip
+                - from: soc.fields.status
+                  to: http.response.status_code
+                - from: soc.fields.method
+                  to: http.request.method
+                - from: soc.fields.path
+                  to: url.path
+                - from: soc.message
+                  to: event.action
+                - from: soc.level
+                  to: log.level
+        tags:
+          - so-soc
+  - id: logfile-logs-906e0d4c-9ec3-4c6a-bef6-e347ec9fd073
+    name: soc-sensoroni-logs
+    revision: 2
+    type: logfile
+    use_output: default
+    meta:
+      package:
+        name: log
+        version:
+    data_stream:
+      namespace: so
+    package_policy_id: 906e0d4c-9ec3-4c6a-bef6-e347ec9fd073
+    streams:
+      - id: logfile-log.log-906e0d4c-9ec3-4c6a-bef6-e347ec9fd073
+        data_stream:
+          dataset: soc
+        pipeline: common
+        paths:
+          - /opt/so/log/sensoroni/sensoroni.log
+        processors:
+          - decode_json_fields:
+              add_error_key: true
+              process_array: true
+              max_depth: 2
+              fields:
+                - message
+              target: sensoroni
+          - add_fields:
+              fields:
                 module: soc
                 dataset_temp: sensoroni
+                category: host
+              target: event
           - rename:
+              ignore_missing: true
               fields:
                 - from: sensoroni.fields.sourceIp
                   to: source.ip
@@ -162,100 +271,141 @@ inputs:
                   to: event.action
                 - from: sensoroni.level
                   to: log.level
-              ignore_missing: true
-        file_identity.native: null
-        prospector.scanner.fingerprint.enabled: false
-  - id: filestream-filestream-976e3900-25ef-11f0-a18d-1b26f69b8310
-    name: suricata-logs
-    revision: 3
-    type: filestream
+  - id: logfile-logs-df0d7f2c-221f-433b-b18b-d1cf83250515
+    name: soc-salt-relay-logs
+    revision: 2
+    type: logfile
     use_output: default
     meta:
       package:
-        name: filestream
+        name: log
         version:
     data_stream:
       namespace: so
-    package_policy_id: 976e3900-25ef-11f0-a18d-1b26f69b8310
+    package_policy_id: df0d7f2c-221f-433b-b18b-d1cf83250515
     streams:
-      - id: filestream-filestream.generic-976e3900-25ef-11f0-a18d-1b26f69b8310
+      - id: logfile-log.log-df0d7f2c-221f-433b-b18b-d1cf83250515
+        data_stream:
+          dataset: soc
+        pipeline: common
+        paths:
+          - /opt/so/log/soc/salt-relay.log
+        processors:
+          - dissect:
+              field: message
+              tokenizer: '%{soc.ts} | %{event.action}'
+              target_prefix: ''
+          - add_fields:
+              fields:
+                module: soc
+                dataset_temp: salt_relay
+                category: host
+              target: event
+        tags:
+          - so-soc
+  - id: logfile-logs-74bd2366-fe52-493c-bddc-843a017fc4d0
+    name: soc-auth-sync-logs
+    revision: 2
+    type: logfile
+    use_output: default
+    meta:
+      package:
+        name: log
+        version:
+    data_stream:
+      namespace: so
+    package_policy_id: 74bd2366-fe52-493c-bddc-843a017fc4d0
+    streams:
+      - id: logfile-log.log-74bd2366-fe52-493c-bddc-843a017fc4d0
+        data_stream:
+          dataset: soc
+        pipeline: common
+        paths:
+          - /opt/so/log/soc/sync.log
+        processors:
+          - dissect:
+              field: message
+              tokenizer: '%{event.action}'
+              target_prefix: ''
+          - add_fields:
+              fields:
+                module: soc
+                dataset_temp: auth_sync
+                category: host
+              target: event
+        tags:
+          - so-soc
+  - id: logfile-logs-d151d9bf-ff2a-4529-9520-c99244bc0253
+    name: suricata-logs
+    revision: 2
+    type: logfile
+    use_output: default
+    meta:
+      package:
+        name: log
+        version:
+    data_stream:
+      namespace: so
+    package_policy_id: d151d9bf-ff2a-4529-9520-c99244bc0253
+    streams:
+      - id: logfile-log.log-d151d9bf-ff2a-4529-9520-c99244bc0253
         data_stream:
           dataset: suricata
+        pipeline: suricata.common
         paths:
           - /nsm/suricata/eve*.json
-        pipeline: suricata.common
-        prospector.scanner.recursive_glob: true
-        prospector.scanner.exclude_files:
-          - \.gz$
-        clean_inactive: -1
-        parsers: null
         processors:
           - add_fields:
-              target: event
               fields:
-                category: network
                 module: suricata
-        file_identity.native: null
-        prospector.scanner.fingerprint.enabled: false
-  - id: filestream-filestream-95091fe0-25ef-11f0-a18d-1b26f69b8310
+                category: network
+              target: event
+  - id: logfile-logs-31f94d05-ae75-40ee-b9c5-0e0356eff327
     name: strelka-logs
-    revision: 3
-    type: filestream
+    revision: 2
+    type: logfile
     use_output: default
     meta:
       package:
-        name: filestream
+        name: log
         version:
     data_stream:
       namespace: so
-    package_policy_id: 95091fe0-25ef-11f0-a18d-1b26f69b8310
+    package_policy_id: 31f94d05-ae75-40ee-b9c5-0e0356eff327
     streams:
-      - id: filestream-filestream.generic-95091fe0-25ef-11f0-a18d-1b26f69b8310
+      - id: logfile-log.log-31f94d05-ae75-40ee-b9c5-0e0356eff327
         data_stream:
           dataset: strelka
+        pipeline: strelka.file
         paths:
           - /nsm/strelka/log/strelka.log
-        pipeline: strelka.file
-        prospector.scanner.recursive_glob: true
-        prospector.scanner.exclude_files:
-          - \.gz$
-        clean_inactive: -1
-        parsers: null
         processors:
           - add_fields:
-              target: event
               fields:
-                category: file
                 module: strelka
-        file_identity.native: null
-        prospector.scanner.fingerprint.enabled: false
-  - id: filestream-filestream-9f309ca0-25ef-11f0-a18d-1b26f69b8310
+                category: file
+              target: event
+  - id: logfile-logs-6197fe84-9b58-4d9b-8464-3d517f28808d
     name: zeek-logs
-    revision: 2
-    type: filestream
+    revision: 1
+    type: logfile
     use_output: default
     meta:
       package:
-        name: filestream
-        version:
+        name: log
+        version: 
     data_stream:
       namespace: so
-    package_policy_id: 9f309ca0-25ef-11f0-a18d-1b26f69b8310
+    package_policy_id: 6197fe84-9b58-4d9b-8464-3d517f28808d
     streams:
-      - id: filestream-filestream.generic-9f309ca0-25ef-11f0-a18d-1b26f69b8310
+      - id: logfile-log.log-6197fe84-9b58-4d9b-8464-3d517f28808d
         data_stream:
           dataset: zeek
         paths:
           - /nsm/zeek/logs/current/*.log
-        prospector.scanner.recursive_glob: true
-        prospector.scanner.exclude_files:
-          - >-
-            (broker|capture_loss|cluster|conn-summary|console|ecat_arp_info|known_certs|known_hosts|known_services|loaded_scripts|ntp|ocsp|packet_filter|reporter|stats|stderr|stdout).log$
-        clean_inactive: -1
-        parsers: null
         processors:
           - dissect:
-              tokenizer: /nsm/zeek/logs/current/%{pipeline}.log
+              tokenizer: '/nsm/zeek/logs/current/%{pipeline}.log'
               field: log.file.path
               trim_chars: .log
               target_prefix: ''
@@ -277,17 +427,18 @@ inputs:
                 regexp:
                   pipeline: >-
                     ^bacnet*|^bsap*|^cip*|^cotp*|^dnp3*|^ecat*|^enip*|^modbus*|^opcua*|^profinet*|^s7comm*
-        file_identity.native: null
-        prospector.scanner.fingerprint.enabled: false
+        exclude_files:
+          - >-
+            broker|capture_loss|cluster|ecat_arp_info|known_hosts|known_services|loaded_scripts|ntp|ocsp|packet_filter|reporter|stats|stderr|stdout.log$
   - id: udp-udp-35051de0-46a5-11ee-8d5d-9f98c8182f60
     name: syslog-udp-514
-    revision: 4
+    revision: 3
     type: udp
     use_output: default
     meta:
       package:
         name: udp
-        version:
+        version: 1.10.0
     data_stream:
       namespace: so
     package_policy_id: 35051de0-46a5-11ee-8d5d-9f98c8182f60
@@ -307,13 +458,13 @@ inputs:
           - syslog
   - id: tcp-tcp-33d37bb0-46a5-11ee-8d5d-9f98c8182f60
     name: syslog-tcp-514
-    revision: 4
+    revision: 3
     type: tcp
     use_output: default
     meta:
       package:
         name: tcp
-        version:
+        version: 1.10.0
     data_stream:
       namespace: so
     package_policy_id: 33d37bb0-46a5-11ee-8d5d-9f98c8182f60

salt/elasticfleet/files/integrations/grid-nodes_general/redis-logs.json

@@ -15,7 +15,7 @@
           "enabled": true,
           "vars": {
             "paths": [
-              "/opt/so/log/redis/redis.log"
+              "/opt/so/log/redis/redis-server.log"
             ],
             "tags": [
               "redis-log"

salt/elasticfleet/files/integrations/grid-nodes_heavy/elasticsearch-grid-nodes.json

@@ -1,107 +0,0 @@
-{
-  "package": {
-    "name": "elasticsearch",
-    "version": ""
-  },
-  "name": "elasticsearch-grid-nodes_heavy",
-  "namespace": "default",
-  "description": "Elasticsearch Logs",
-  "policy_id": "so-grid-nodes_heavy",
-  "inputs": {
-    "elasticsearch-logfile": {
-      "enabled": true,
-      "streams": {
-        "elasticsearch.audit": {
-          "enabled": false,
-          "vars": {
-            "paths": [
-              "/var/log/elasticsearch/*_audit.json"
-            ]
-          }
-        },
-        "elasticsearch.deprecation": {
-          "enabled": false,
-          "vars": {
-            "paths": [
-              "/var/log/elasticsearch/*_deprecation.json"
-            ]
-          }
-        },
-        "elasticsearch.gc": {
-          "enabled": false,
-          "vars": {
-            "paths": [
-              "/var/log/elasticsearch/gc.log.[0-9]*",
-              "/var/log/elasticsearch/gc.log"
-            ]
-          }
-        },
-        "elasticsearch.server": {
-          "enabled": true,
-          "vars": {
-            "paths": [
-              "/opt/so/log/elasticsearch/*.json"
-            ]
-          }
-        },
-        "elasticsearch.slowlog": {
-          "enabled": false,
-          "vars": {
-            "paths": [
-              "/var/log/elasticsearch/*_index_search_slowlog.json",
-              "/var/log/elasticsearch/*_index_indexing_slowlog.json"
-            ]
-          }
-        }
-      }
-    },
-    "elasticsearch-elasticsearch/metrics": {
-      "enabled": false,
-      "vars": {
-        "hosts": [
-          "http://localhost:9200"
-        ],
-        "scope": "node"
-      },
-      "streams": {
-        "elasticsearch.stack_monitoring.ccr": {
-          "enabled": false
-        },
-        "elasticsearch.stack_monitoring.cluster_stats": {
-          "enabled": false
-        },
-        "elasticsearch.stack_monitoring.enrich": {
-          "enabled": false
-        },
-        "elasticsearch.stack_monitoring.index": {
-          "enabled": false
-        },
-        "elasticsearch.stack_monitoring.index_recovery": {
-          "enabled": false,
-          "vars": {
-            "active.only": true
-          }
-        },
-        "elasticsearch.stack_monitoring.index_summary": {
-          "enabled": false
-        },
-        "elasticsearch.stack_monitoring.ml_job": {
-          "enabled": false
-        },
-        "elasticsearch.stack_monitoring.node": {
-          "enabled": false
-        },
-        "elasticsearch.stack_monitoring.node_stats": {
-          "enabled": false
-        },
-        "elasticsearch.stack_monitoring.pending_tasks": {
-          "enabled": false
-        },
-        "elasticsearch.stack_monitoring.shard": {
-          "enabled": false
-        }
-      }
-    }
-  },
-  "force": true
-}

salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load

@@ -17,9 +17,9 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then
 
   # Third, configure Elastic Defend Integration seperately
   /usr/sbin/so-elastic-fleet-integration-policy-elastic-defend
-
   # Initial Endpoints
-  for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/endpoints-initial/*.json; do
+  for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/endpoints-initial/*.json
+  do
     printf "\n\nInitial Endpoints Policy - Loading $INTEGRATION\n"
     elastic_fleet_integration_check "endpoints-initial" "$INTEGRATION"
     if [ -n "$INTEGRATION_ID" ]; then
@@ -40,7 +40,8 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then
   done
 
   # Grid Nodes - General
-  for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/grid-nodes_general/*.json; do
+  for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/grid-nodes_general/*.json
+  do
     printf "\n\nGrid Nodes Policy_General - Loading $INTEGRATION\n"
     elastic_fleet_integration_check "so-grid-nodes_general" "$INTEGRATION"
     if [ -n "$INTEGRATION_ID" ]; then
@@ -59,9 +60,13 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then
       fi
     fi
   done
+  if [[ "$RETURN_CODE" != "1" ]]; then
+    touch /opt/so/state/eaintegrations.txt
+  fi
 
   # Grid Nodes - Heavy
-  for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/grid-nodes_heavy/*.json; do
+  for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/grid-nodes_heavy/*.json
+  do
     printf "\n\nGrid Nodes Policy_Heavy - Loading $INTEGRATION\n"
     elastic_fleet_integration_check "so-grid-nodes_heavy" "$INTEGRATION"
     if [ -n "$INTEGRATION_ID" ]; then
@@ -73,16 +78,22 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then
       fi
     else
       printf "\n\nIntegration does not exist - Creating integration\n"
-      if ! elastic_fleet_integration_create "@$INTEGRATION"; then
-        echo -e "\nFailed to create integration for ${INTEGRATION##*/}"
-        RETURN_CODE=1
-        continue
+      if [ "$NAME" != "elasticsearch-logs" ]; then
+        if ! elastic_fleet_integration_create "@$INTEGRATION"; then
+          echo -e "\nFailed to create integration for ${INTEGRATION##*/}"
+          RETURN_CODE=1
+          continue
+        fi
       fi
     fi
   done
+  if [[ "$RETURN_CODE" != "1" ]]; then
+    touch /opt/so/state/eaintegrations.txt
+  fi
 
   # Fleet Server - Optional integrations
-  for INTEGRATION in /opt/so/conf/elastic-fleet/integrations-optional/FleetServer*/*.json; do
+  for INTEGRATION in /opt/so/conf/elastic-fleet/integrations-optional/FleetServer*/*.json
+  do
     if ! [ "$INTEGRATION" == "/opt/so/conf/elastic-fleet/integrations-optional/FleetServer*/*.json" ]; then
       FLEET_POLICY=`echo "$INTEGRATION"| cut -d'/' -f7`
       printf "\n\nFleet Server Policy - Loading $INTEGRATION\n"
@@ -106,8 +117,6 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then
       fi
     fi
   done
-
-  # Only create the state file if all policies were created/updated successfully
   if [[ "$RETURN_CODE" != "1" ]]; then
     touch /opt/so/state/eaintegrations.txt
   fi

salt/elasticsearch/defaults.yaml

@@ -691,6 +691,7 @@ elasticsearch:
                 match_mapping_type: string
           settings:
             index:
+              final_pipeline: .fleet_final_pipeline-1
               lifecycle:
                 name: so-import-logs
               mapping:

salt/elasticsearch/files/ingest/global@custom

@@ -1,212 +1,31 @@
 {
-    "version": 3,
-    "_meta": {
-        "managed_by": "securityonion",
-        "managed": true
-    },
-    "description": "Custom pipeline for processing all incoming Fleet Agent documents. \n",
-    "processors": [
-        {
-            "set": {
-                "ignore_failure": true,
-                "field": "event.module",
-                "value": "elastic_agent"
-            }
-        },
-        {
-            "split": {
-                "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')",
-                "field": "event.dataset",
-                "separator": "\\.",
-                "target_field": "module_temp"
-            }
-        },
-        {
-            "split": {
-                "if": "ctx.data_stream?.dataset != null && ctx.data_stream?.dataset.contains('.')",
-                "field": "data_stream.dataset",
-                "separator": "\\.",
-                "target_field": "datastream_dataset_temp",
-                "ignore_missing": true
-            }
-        },
-        {
-            "set": {
-                "if": "ctx.module_temp != null",
-                "override": true,
-                "field": "event.module",
-                "value": "{{module_temp.0}}"
-            }
-        },
-        {
-            "set": {
-                "if": "ctx.datastream_dataset_temp != null && ctx.datastream_dataset_temp[0] == 'network_traffic'",
-                "field": "event.module",
-                "value": "{{ datastream_dataset_temp.0 }}",
-                "ignore_failure": true,
-                "ignore_empty_value": true,
-                "description": "Fix EA network packet capture"
-            }
-        },
-        {
-            "gsub": {
-                "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')",
-                "field": "event.dataset",
-                "pattern": "^[^.]*.",
-                "replacement": "",
-                "target_field": "dataset_tag_temp"
-            }
-        },
-        {
-            "append": {
-                "if": "ctx.dataset_tag_temp != null",
-                "field": "tags",
-                "value": "{{dataset_tag_temp}}",
-                "allow_duplicates": false
-            }
-        },
-        {
-            "set": {
-                "if": "ctx.network?.direction == 'egress'",
-                "override": true,
-                "field": "network.initiated",
-                "value": "true"
-            }
-        },
-        {
-            "set": {
-                "if": "ctx.network?.direction == 'ingress'",
-                "override": true,
-                "field": "network.initiated",
-                "value": "false"
-            }
-        },
-        {
-            "set": {
-                "if": "ctx.network?.type == 'ipv4'",
-                "override": true,
-                "field": "destination.ipv6",
-                "value": "false"
-            }
-        },
-        {
-            "set": {
-                "if": "ctx.network?.type == 'ipv6'",
-                "override": true,
-                "field": "destination.ipv6",
-                "value": "true"
-            }
-        },
-        {
-            "set": {
-                "if": "ctx.tags != null && ctx.tags.contains('import')",
-                "override": true,
-                "field": "data_stream.dataset",
-                "value": "import"
-            }
-        },
-        {
-            "set": {
-                "if": "ctx.tags != null && ctx.tags.contains('import')",
-                "override": true,
-                "field": "data_stream.namespace",
-                "value": "so"
-            }
-        },
-        {
-            "community_id": {
-                "if": "ctx.event?.dataset == 'endpoint.events.network'",
-                "ignore_failure": true
-            }
-        },
-        {
-            "set": {
-                "if": "ctx.event?.module == 'fim'",
-                "override": true,
-                "field": "event.module",
-                "value": "file_integrity"
-            }
-        },
-        {
-            "rename": {
-                "if": "ctx.winlog?.provider_name == 'Microsoft-Windows-Windows Defender'",
-                "ignore_missing": true,
-                "field": "winlog.event_data.Threat Name",
-                "target_field": "winlog.event_data.threat_name"
-            }
-        },
-        {
-            "set": {
-                "if": "ctx?.metadata?.kafka != null",
-                "field": "kafka.id",
-                "value": "{{metadata.kafka.partition}}{{metadata.kafka.offset}}{{metadata.kafka.timestamp}}",
-                "ignore_failure": true
-            }
-        },
-        {
-            "set": {
-                "if": "ctx.event?.dataset != null && ctx.event?.dataset == 'elasticsearch.server'",
-                "field": "event.module",
-                "value": "elasticsearch"
-            }
-        },
-        {
-            "append": {
-                "field": "related.ip",
-                "value": [
-                    "{{source.ip}}",
-                    "{{destination.ip}}"
-                ],
-                "allow_duplicates": false,
-                "if": "ctx?.event?.dataset == 'endpoint.events.network' && ctx?.source?.ip != null",
-                "ignore_failure": true
-            }
-        },
-        {
-            "foreach": {
-                "field": "host.ip",
-                "processor": {
-                    "append": {
-                        "field": "related.ip",
-                        "value": "{{_ingest._value}}",
-                        "allow_duplicates": false
-                    }
-                },
-                "if": "ctx?.event?.module == 'endpoint' && ctx?.host?.ip != null",
-                "ignore_missing": true,
-                "description": "Extract IPs from Elastic Agent events (host.ip) and adds them to related.ip"
-            }
-        },
-        {
-            "pipeline": {
-                "name": ".fleet_final_pipeline-1",
-                "ignore_missing_pipeline": true
-            }
-        },
-        {
-            "remove": {
-                "field": "event.agent_id_status",
-                "ignore_missing": true,
-                "if": "ctx?.event?.agent_id_status == 'auth_metadata_missing'"
-            }
-        },
-        {
-            "remove": {
-                "field": [
-                    "message2",
-                    "type",
-                    "fields",
-                    "category",
-                    "module",
-                    "dataset",
-                    "event.dataset_temp",
-                    "dataset_tag_temp",
-                    "module_temp",
-                    "datastream_dataset_temp"
-                ],
-                "ignore_missing": true,
-                "ignore_failure": true
-            }
-        }
-    ]
-}
+  "version": 3,
+  "_meta": {
+    "managed_by": "securityonion",
+    "managed": true
+  },
+  "description": "Custom pipeline for processing all incoming Fleet Agent documents. \n",
+  "processors": [
+    { "set":        { "ignore_failure": true,  "field": "event.module", "value": "elastic_agent" } },
+    { "split":      { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "separator": "\\.", "target_field": "module_temp"  } },
+    { "split":      { "if": "ctx.data_stream?.dataset != null && ctx.data_stream?.dataset.contains('.')", "field":"data_stream.dataset", "separator":"\\.", "target_field":"datastream_dataset_temp", "ignore_missing":true } },
+    { "set":        { "if": "ctx.module_temp != null", "override": true, "field": "event.module", "value": "{{module_temp.0}}" } },
+    { "set":        { "if": "ctx.datastream_dataset_temp != null && ctx.datastream_dataset_temp[0] == 'network_traffic'", "field":"event.module", "value":"{{ datastream_dataset_temp.0 }}", "ignore_failure":true, "ignore_empty_value":true, "description":"Fix EA network packet capture" } },
+    { "gsub":       { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "pattern": "^[^.]*.", "replacement": "", "target_field": "dataset_tag_temp" } },
+    { "append":     { "if": "ctx.dataset_tag_temp != null", "field": "tags", "value": "{{dataset_tag_temp}}", "allow_duplicates": false  } },
+    { "set":        { "if": "ctx.network?.direction == 'egress'", "override": true, "field": "network.initiated", "value": "true" } },
+    { "set":        { "if": "ctx.network?.direction == 'ingress'", "override": true, "field": "network.initiated", "value": "false" } },
+    { "set":        { "if": "ctx.network?.type == 'ipv4'", "override": true, "field": "destination.ipv6", "value": "false" } },
+    { "set":        { "if": "ctx.network?.type == 'ipv6'", "override": true, "field": "destination.ipv6", "value": "true" } },
+    { "set":        { "if": "ctx.tags != null && ctx.tags.contains('import')", "override": true, "field": "data_stream.dataset", "value": "import" } },
+    { "set":        { "if": "ctx.tags != null && ctx.tags.contains('import')", "override": true, "field": "data_stream.namespace", "value": "so" } },
+    { "community_id":{ "if": "ctx.event?.dataset == 'endpoint.events.network'", "ignore_failure":true }  },
+    { "set":        { "if": "ctx.event?.module == 'fim'", "override": true, "field": "event.module", "value": "file_integrity" } },
+    { "rename":       { "if": "ctx.winlog?.provider_name == 'Microsoft-Windows-Windows Defender'", "ignore_missing": true, "field": "winlog.event_data.Threat Name", "target_field": "winlog.event_data.threat_name" } },
+    { "set":        { "if": "ctx?.metadata?.kafka != null" , "field": "kafka.id", "value": "{{metadata.kafka.partition}}{{metadata.kafka.offset}}{{metadata.kafka.timestamp}}", "ignore_failure": true } },
+    { "set":        { "if": "ctx.event?.dataset != null && ctx.event?.dataset == 'elasticsearch.server'", "field": "event.module", "value":"elasticsearch" }},
+    {"append":      {"field":"related.ip","value":["{{source.ip}}","{{destination.ip}}"],"allow_duplicates":false,"if":"ctx?.event?.dataset == 'endpoint.events.network' && ctx?.source?.ip != null","ignore_failure":true}},
+    {"foreach":     {"field":"host.ip","processor":{"append":{"field":"related.ip","value":"{{_ingest._value}}","allow_duplicates":false}},"if":"ctx?.event?.module == 'endpoint' && ctx?.host?.ip != null","ignore_missing":true, "description":"Extract IPs from Elastic Agent events (host.ip) and adds them to related.ip"}},
+    { "remove":     { "field": [ "message2", "type", "fields", "category", "module", "dataset", "event.dataset_temp", "dataset_tag_temp", "module_temp", "datastream_dataset_temp" ], "ignore_missing": true, "ignore_failure": true } }
+  ]
+}

salt/elasticsearch/templates/component/elastic-agent/so-fleet_agent_id_verification-1.json

@@ -2,7 +2,7 @@
   "template": {
     "settings": {
       "index": {
-        "final_pipeline": "global@custom"
+        "final_pipeline": ".fleet_final_pipeline-1"
       }
     },
     "mappings": {

salt/manager/tools/sbin/so-minion

@@ -839,6 +839,7 @@ function createHEAVYNODE() {
 	add_elastic_agent_to_minion || return 1
 	add_sensor_to_minion || return 1
 	add_strelka_to_minion || return 1
+	add_redis_to_minion || return 1
 	add_telegraf_to_minion || return 1
 }
 

salt/manager/tools/sbin/soup

@@ -353,22 +353,6 @@ disable_logstash_heavynodes() {
   done
 }
 
-disable_redis_heavynodes() {
-  local c=0
-  printf "\nChecking for heavynodes and disabling Redis if they exist\n"
-  for file in /opt/so/saltstack/local/pillar/minions/*.sls; do
-    if [[ "$file" =~ "_heavynode.sls" && ! "$file" =~ "/opt/so/saltstack/local/pillar/minions/adv_" ]]; then
-      c=1
-      echo "Disabling Redis for: $file"
-      so-yaml.py replace "$file" redis.enabled False
-    fi
-  done
-
-  if [[ "$c" != 0 ]]; then
-    FINAL_MESSAGE_QUEUE+=("Redis has been disabled on all heavynodes.")
-  fi
-}
-
 enable_highstate() {
     echo "Enabling highstate."
     salt-call state.enable highstate -l info --local
@@ -690,8 +674,6 @@ post_to_2.4.210() {
 
   rollover_index "logs-kratos-so"
 
-  disable_redis_heavynodes
-
   echo "Regenerating Elastic Agent Installers"
   /sbin/so-elastic-agent-gen-installers
 

salt/telegraf/etc/telegraf.conf

@@ -7,7 +7,6 @@
 {%- set UNIQUEID = salt['pillar.get']('sensor:uniqueid', '') %}
 {%- set ZEEK_ENABLED = salt['pillar.get']('zeek:enabled', True) %}
 {%- set MDENGINE = GLOBALS.md_engine %}
-{%- set LOGSTASH_ENABLED = salt['pillar.get']('logstash:enabled', False) %}
 # Global tags can be specified here in key="value" format.
 [global_tags]
   role = "{{ GLOBALS.role.split('-') | last }}"
@@ -242,8 +241,12 @@
 #   ## Use TLS but skip chain & host verification
 #   # insecure_skip_verify = false
 
-{#- Fleet nodes do not have pillar access to logstash credentials #}
-{%- if LOGSTASH_ENABLED and grains.role != 'so-fleet' %}
+{%- set logstash_metrics_roles = ['so-searchnode','so-standalone','so-managersearch','so-heavynode'] %}
+{%- if GLOBALS.pipeline != "KAFKA" %}
+{%-   set logstash_metrics_roles = logstash_metrics_roles + ['so-manager', 'so-receiver'] %}
+{%- endif %}
+
+{%- if grains.role in logstash_metrics_roles %}
 [[inputs.logstash]]
   url = "http://localhost:9600"
   collect = ["pipelines"]