From d8be6e42e13a427e3acaa9d02f7a8f72bc908691 Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Mon, 7 Jul 2025 12:58:00 -0500
Subject: [PATCH 01/31] es 8.18.3

---
 VERSION                          | 2 +-
 salt/elasticsearch/defaults.yaml | 2 +-
 salt/kibana/defaults.yaml        | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/VERSION b/VERSION
index 032d0bb01..452820224 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-2.4.170
+2.4.0-foxtrot
\ No newline at end of file
diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml
index 1f81e95d2..73c29c8c6 100644
--- a/salt/elasticsearch/defaults.yaml
+++ b/salt/elasticsearch/defaults.yaml
@@ -1,6 +1,6 @@
 elasticsearch:
   enabled: false
-  version: 8.17.3
+  version: 8.18.3
   index_clean: true
   config:
     action:
diff --git a/salt/kibana/defaults.yaml b/salt/kibana/defaults.yaml
index 2446821f1..a636f9b79 100644
--- a/salt/kibana/defaults.yaml
+++ b/salt/kibana/defaults.yaml
@@ -22,7 +22,7 @@ kibana:
           - default
           - file
     migrations:
-      discardCorruptObjects: "8.17.3"
+      discardCorruptObjects: "8.18.3"
     telemetry:
       enabled: False
     security:

From 4bf2c931e9e7279b64f79da589c6000a58506d14 Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Tue, 8 Jul 2025 08:43:24 -0500
Subject: [PATCH 02/31] make sure required file exists to generate
 ADDON_INTEGRATION_DEFAULTS

---
 salt/manager/managed_soc_annotations.sls | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/manager/managed_soc_annotations.sls b/salt/manager/managed_soc_annotations.sls
index 17621f973..9ef0b1ea1 100644
--- a/salt/manager/managed_soc_annotations.sls
+++ b/salt/manager/managed_soc_annotations.sls
@@ -5,7 +5,7 @@
 
 {# Managed elasticsearch/soc_elasticsearch.yaml file for adding integration configuration items to UI #}
 {% set managed_integrations = salt['pillar.get']('elasticsearch:managed_integrations', []) %}
-{% if managed_integrations %}
+{% if managed_integrations and salt['file.file_exists']('/opt/so/state/esfleet_package_components.json') %}
 {%   from 'elasticfleet/integration-defaults.map.jinja' import ADDON_INTEGRATION_DEFAULTS %}
 {%   set addon_integration_keys = ADDON_INTEGRATION_DEFAULTS.keys() %}
 {%   set matched_integration_names = [] %}

From 42552810fb1ce3b217deef9573495becce365044 Mon Sep 17 00:00:00 2001
From: Josh Brower <Josh@defensivedepth.com>
Date: Tue, 8 Jul 2025 09:50:08 -0400
Subject: [PATCH 03/31] Add user.name to kratos query

---
 salt/soc/defaults.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml
index 42e07a7c0..cb12671f8 100644
--- a/salt/soc/defaults.yaml
+++ b/salt/soc/defaults.yaml
@@ -1910,7 +1910,7 @@ soc:
               query: 'event.dataset:kratos.audit AND msg:*authenticated* | groupby http_request.headers.x-real-ip | groupby -sankey http_request.headers.x-real-ip user.name | groupby user.name | groupby http_request.headers.user-agent'
             - name: SOC Login Failures
               description: SOC (Security Onion Console) login failures
-              query: 'event.dataset:kratos.audit AND msg:*Encountered*self-service*login*error* | groupby http_request.headers.x-real-ip | groupby -sankey http_request.headers.x-real-ip http_request.headers.user-agent | groupby http_request.headers.user-agent'
+              query: 'event.dataset:kratos.audit AND msg:*Encountered*self-service*login*error* | groupby user.name | groupby http_request.headers.x-real-ip | groupby -sankey http_request.headers.x-real-ip http_request.headers.user-agent | groupby http_request.headers.user-agent'
             - name: Alerts
               description: Overview of all alerts
               query: 'tags:alert | groupby event.module* | groupby source.ip | groupby -sankey source.ip destination.ip | groupby destination.ip | groupby destination.port | groupby rule.name | groupby event.severity | groupby destination_geo.organization_name'

From d4f0cbcb677b80407e70d940d7a5a4f153429495 Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Tue, 8 Jul 2025 15:23:46 -0500
Subject: [PATCH 04/31] changes for 'generic' integrations with no compoent
 templates assigned. Default to using the logs-filestream.generic@package
 componet template

---
 .../integration-defaults.map.jinja            | 132 ++++++++++--------
 ...o-elastic-fleet-optional-integrations-load |   4 +
 .../so-elasticsearch-templates-load           |   2 +-
 3 files changed, 81 insertions(+), 57 deletions(-)

diff --git a/salt/elasticfleet/integration-defaults.map.jinja b/salt/elasticfleet/integration-defaults.map.jinja
index 5449e791e..62e37b229 100644
--- a/salt/elasticfleet/integration-defaults.map.jinja
+++ b/salt/elasticfleet/integration-defaults.map.jinja
@@ -4,6 +4,7 @@
 
 
 {% import_json '/opt/so/state/esfleet_package_components.json' as ADDON_PACKAGE_COMPONENTS %}
+{% import_json '/opt/so/state/esfleet_package_components.json' as INSTALLED_COMPONENT_TEMPLATES %}
 {% import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %}
 
 {% set CORE_ESFLEET_PACKAGES = ELASTICFLEETDEFAULTS.get('elasticfleet', {}).get('packages', {}) %}
@@ -62,70 +63,89 @@
 {%         else %}
 {%           set integration_type = "" %}
 {%         endif %}
-{%           set component_name = pkg.name ~ "." ~ pattern.title %}
-{#           fix weirdly named components #}
-{%           if component_name in WEIRD_INTEGRATIONS %}
-{%             set component_name = WEIRD_INTEGRATIONS[component_name] %}
-{%           endif %}
+{%         set component_name = pkg.name ~ "." ~ pattern.title %}
+{%         set index_pattern = pattern.name %}
+
+{#         fix weirdly named components #}
+{%         if component_name in WEIRD_INTEGRATIONS %}
+{%           set component_name = WEIRD_INTEGRATIONS[component_name] %}
+{%         endif %}
+
+{#         create duplicate of component_name, so we can split generics from @custom component templates in the index template below and overwrite the default @package when needed
+           eg. having to replace unifiedlogs.generic@package with filestream.generic@package, but keep the ability to customize unifiedlogs.generic@custom and its ILM policy #}
+{%         set custom_component_name = component_name %}
+
+{#         duplicate integration_type to assist with sometimes needing to overwrite component templates with 'logs-filestream.generic@package' (there is no metrics-filestream.generic@package) #}
+{%         set generic_integration_type = integration_type %}
+
 {#           component_name_x maintains the functionality of merging local pillar changes with generated 'defaults' via SOC UI #}
 {%           set component_name_x = component_name.replace(".","_x_") %}
 {#           pillar overrides/merge expects the key names to follow the naming in elasticsearch/defaults.yaml eg. so-logs-1password_x_item_usages . The _x_ is replaced later on in elasticsearch/template.map.jinja #}
 {%           set integration_key = "so-" ~ integration_type ~ component_name_x %}
 
+{%         if ".generic" in component_name and component_name not in INSTALLED_COMPONENT_TEMPLATES %}
+{#           these generic templates by default are directed to index_pattern of 'logs-generic-*', overwrite that here to point to eg gcp_pubsub.generic-* #}
+{%           set index_pattern = integration_type ~ component_name ~ "-*" %}
+{#           includes use of .generic component template, but it doesn't exist in installed component templates. Redirect it to filestream.generic@package #}
+{%           set component_name = "filestream.generic" %}
+{%           set generic_integration_type = "logs-" %}
+{%         endif %}
+
 {#           Default integration settings #}
 {%           set integration_defaults = {
-              "index_sorting": false,
-              "index_template": {
-                  "composed_of": [integration_type ~ component_name ~ "@package", integration_type ~ component_name ~ "@custom", "so-fleet_integrations.ip_mappings-1", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"],
-                  "data_stream": {
-                      "allow_custom_routing": false,
-                      "hidden": false
-                  },
-                  "ignore_missing_component_templates": [integration_type ~ component_name ~ "@custom"],
-                  "index_patterns": [pattern.name],
-                  "priority": 501,
-                  "template": {
-                      "settings": {
-                          "index": {
-                              "lifecycle": {"name": "so-" ~ integration_type ~ component_name ~ "-logs"},
-                              "number_of_replicas": 0
-                          }
-                      }
-                  }
-              },
-              "policy": {
-                  "phases": {
-                      "cold": {
-                          "actions": {
-                              "set_priority": {"priority": 0}
-                          },
-                          "min_age": "60d"
+               "index_sorting": false,
+               "index_template": {
+                 "composed_of": [generic_integration_type ~ component_name ~ "@package", integration_type ~ custom_component_name ~ "@custom", "so-fleet_integrations.ip_mappings-1", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"],
+                 "data_stream": {
+                   "allow_custom_routing": false,
+                   "hidden": false
+                 },
+                 "ignore_missing_component_templates": [integration_type ~ custom_component_name ~ "@custom"],
+                 "index_patterns": [index_pattern],
+                 "priority": 501,
+                 "template": {
+                   "settings": {
+                     "index": {
+                       "lifecycle": {"name": "so-" ~ integration_type ~ custom_component_name ~ "-logs"},
+                         "number_of_replicas": 0
+                     }
+                   }
+                 }
+               },
+               "policy": {
+                 "phases": {
+                   "cold": {
+                     "actions": {
+                       "set_priority": {"priority": 0}
+                     },
+                     "min_age": "60d"
+                   },
+                   "delete": {
+                     "actions": {
+                       "delete": {}
+                     },
+                     "min_age": "365d"
+                   },
+                   "hot": {
+                     "actions": {
+                       "rollover": {
+                         "max_age": "30d",
+                         "max_primary_shard_size": "50gb"
+                       },
+                       "set_priority": {"priority": 100}
                       },
-                      "delete": {
-                          "actions": {
-                              "delete": {}
-                          },
-                          "min_age": "365d"
-                      },
-                      "hot": {
-                          "actions": {
-                              "rollover": {
-                                  "max_age": "30d",
-                                  "max_primary_shard_size": "50gb"
-                              },
-                              "set_priority": {"priority": 100}
-                          },
-                          "min_age": "0ms"
-                      },
-                      "warm": {
-                          "actions": {
-                              "set_priority": {"priority": 50}
-                          },
-                          "min_age": "30d"
-                      }
-                  }
-              }
-          } %}
+                      "min_age": "0ms"
+                   },
+                   "warm": {
+                     "actions": {
+                       "set_priority": {"priority": 50}
+                     },
+                     "min_age": "30d"
+                   }
+                 }
+               }
+             } %}
+
 {%         do ADDON_INTEGRATION_DEFAULTS.update({integration_key: integration_defaults}) %}
 {%       endfor %}
 {%     endif %}
diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load
index 26d775e82..df01608f5 100644
--- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load
+++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load
@@ -19,6 +19,7 @@ BULK_INSTALL_PACKAGE_LIST=/tmp/esfleet_bulk_install.json
 BULK_INSTALL_PACKAGE_TMP=/tmp/esfleet_bulk_install_tmp.json
 BULK_INSTALL_OUTPUT=/opt/so/state/esfleet_bulk_install_results.json
 PACKAGE_COMPONENTS=/opt/so/state/esfleet_package_components.json
+COMPONENT_TEMPLATES=/opt/so/state/esfleet_component_templates.txt
 
 PENDING_UPDATE=false
 
@@ -155,6 +156,9 @@ if [[ -f $STATE_FILE_SUCCESS  ]]; then
         # Write out file for generating index/component/ilm templates
         latest_installed_package_list=$(elastic_fleet_installed_packages)
         echo $latest_installed_package_list | jq '[.items[] | {name: .name, es_index_patterns: .dataStreams}]' > $PACKAGE_COMPONENTS
+        # Refresh installed component template list
+        latest_component_templates_list=$(so-elasticsearch-query _component_template | jq '.component_templates[] | .name' | jq -s '.')
+        echo $latest_component_templates_list > $COMPONENT_TEMPLATES
 
     else
         # This is the installation of add-on integrations and upgrade of existing integrations. Exiting without error, next highstate will attempt to re-run.
diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load
index 758f601eb..ba5767255 100755
--- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load
+++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load
@@ -136,7 +136,7 @@ if [ ! -f $STATE_FILE_SUCCESS ]; then
       TEMPLATE=${i::-14}
       COMPONENT_PATTERN=${TEMPLATE:3}
       MATCH=$(echo "$TEMPLATE" | grep -E "^so-logs-|^so-metrics" | grep -vE "detections|osquery")
-      if [[ -n "$MATCH" && ! "$COMPONENT_LIST" =~ "$COMPONENT_PATTERN" && ! "$COMPONENT_PATTERN" =~ logs-http_endpoint\.generic|logs-winlog\.winlog ]]; then
+      if [[ -n "$MATCH" && ! "$COMPONENT_LIST" =~ "$COMPONENT_PATTERN" && ! "$COMPONENT_PATTERN" =~ *\.generic|logs-winlog\.winlog ]]; then
         load_failures=$((load_failures+1))
         echo "Component template does not exist for $COMPONENT_PATTERN. The index template will not be loaded. Load failures: $load_failures"
       else

From bef2fa9e8d5b06c13f045cb7436da05e16c64b1b Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Tue, 8 Jul 2025 16:09:16 -0500
Subject: [PATCH 05/31] 8.18.3 pipeline updates

---
 .../grid-nodes_general/import-evtx-logs.json  |  2 +-
 ...nse.log-1.21.0 => logs-pfsense.log-1.23.0} | 28 +++++++------------
 ...icata => logs-pfsense.log-1.23.0-suricata} |  0
 3 files changed, 11 insertions(+), 19 deletions(-)
 rename salt/elasticsearch/files/ingest/{logs-pfsense.log-1.21.0 => logs-pfsense.log-1.23.0} (93%)
 rename salt/elasticsearch/files/ingest/{logs-pfsense.log-1.21.0-suricata => logs-pfsense.log-1.23.0-suricata} (100%)

diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json
index 46717f3e1..415e333e7 100644
--- a/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json
+++ b/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json
@@ -20,7 +20,7 @@
             ],
             "data_stream.dataset": "import",
             "custom": "",
-            "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- drop_fields:\n    fields: [\"host\"]\n    ignore_missing: true\n- add_fields:\n    target: data_stream\n    fields:\n      type: logs\n      dataset: system.security\n- add_fields:\n    target: event\n    fields:\n      dataset: system.security\n      module: system\n      imported: true\n- add_fields:\n    target: \"@metadata\"\n    fields:\n        pipeline: logs-system.security-1.67.0\n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.sysmon_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.sysmon_operational\n          module: windows\n          imported: true\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.sysmon_operational-2.5.0\n- if:\n    equals:\n      winlog.channel: 'Application'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.application-1.67.0\n- if:\n    equals:\n      winlog.channel: 'System'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.system-1.67.0\n    \n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.powershell_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell_operational\n          module: windows\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.powershell_operational-2.5.0\n- add_fields:\n    target: data_stream\n    fields:\n        dataset: import",
+            "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- drop_fields:\n    fields: [\"host\"]\n    ignore_missing: true\n- add_fields:\n    target: data_stream\n    fields:\n      type: logs\n      dataset: system.security\n- add_fields:\n    target: event\n    fields:\n      dataset: system.security\n      module: system\n      imported: true\n- add_fields:\n    target: \"@metadata\"\n    fields:\n        pipeline: logs-system.security-2.3.1\n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.sysmon_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.sysmon_operational\n          module: windows\n          imported: true\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.sysmon_operational-3.0.0\n- if:\n    equals:\n      winlog.channel: 'Application'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.application-2.3.1\n- if:\n    equals:\n      winlog.channel: 'System'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.system-2.3.1\n    \n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.powershell_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell_operational\n          module: windows\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.powershell_operational-3.0.0\n- add_fields:\n    target: data_stream\n    fields:\n        dataset: import",
             "tags": [
               "import"
             ]
diff --git a/salt/elasticsearch/files/ingest/logs-pfsense.log-1.21.0 b/salt/elasticsearch/files/ingest/logs-pfsense.log-1.23.0
similarity index 93%
rename from salt/elasticsearch/files/ingest/logs-pfsense.log-1.21.0
rename to salt/elasticsearch/files/ingest/logs-pfsense.log-1.23.0
index 7c4f2575f..e79b91b26 100644
--- a/salt/elasticsearch/files/ingest/logs-pfsense.log-1.21.0
+++ b/salt/elasticsearch/files/ingest/logs-pfsense.log-1.23.0
@@ -107,61 +107,61 @@
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.21.0-firewall",
+        "name": "logs-pfsense.log-1.23.0-firewall",
         "if": "ctx.event.provider == 'filterlog'"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.21.0-openvpn",
+        "name": "logs-pfsense.log-1.23.0-openvpn",
         "if": "ctx.event.provider == 'openvpn'"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.21.0-ipsec",
+        "name": "logs-pfsense.log-1.23.0-ipsec",
         "if": "ctx.event.provider == 'charon'"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.21.0-dhcp",
+        "name": "logs-pfsense.log-1.23.0-dhcp",
         "if": "[\"dhcpd\", \"dhclient\", \"dhcp6c\"].contains(ctx.event.provider)"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.21.0-unbound",
+        "name": "logs-pfsense.log-1.23.0-unbound",
         "if": "ctx.event.provider == 'unbound'"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.21.0-haproxy",
+        "name": "logs-pfsense.log-1.23.0-haproxy",
         "if": "ctx.event.provider == 'haproxy'"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.21.0-php-fpm",
+        "name": "logs-pfsense.log-1.23.0-php-fpm",
         "if": "ctx.event.provider == 'php-fpm'"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.21.0-squid",
+        "name": "logs-pfsense.log-1.23.0-squid",
         "if": "ctx.event.provider == 'squid'"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.21.0-snort",
+        "name": "logs-pfsense.log-1.23.0-snort",
         "if": "ctx.event.provider == 'snort'"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.21.0-suricata",
+        "name": "logs-pfsense.log-1.23.0-suricata",
         "if": "ctx.event.provider == 'suricata'"
       }
     },
@@ -358,14 +358,6 @@
         "source": "void handleMap(Map map) {\n  for (def x : map.values()) {\n    if (x instanceof Map) {\n        handleMap(x);\n    } else if (x instanceof List) {\n        handleList(x);\n    }\n  }\n  map.values().removeIf(v -> v == null || (v instanceof String && v == \"-\"));\n}\nvoid handleList(List list) {\n  for (def x : list) {\n      if (x instanceof Map) {\n          handleMap(x);\n      } else if (x instanceof List) {\n          handleList(x);\n      }\n  }\n}\nhandleMap(ctx);\n"
       }
     },
-    {
-      "remove": {
-        "field": "event.original",
-        "if": "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))",
-        "ignore_failure": true,
-        "ignore_missing": true
-      }
-    },
     {
       "pipeline": {
         "name": "global@custom",
diff --git a/salt/elasticsearch/files/ingest/logs-pfsense.log-1.21.0-suricata b/salt/elasticsearch/files/ingest/logs-pfsense.log-1.23.0-suricata
similarity index 100%
rename from salt/elasticsearch/files/ingest/logs-pfsense.log-1.21.0-suricata
rename to salt/elasticsearch/files/ingest/logs-pfsense.log-1.23.0-suricata

From 59628ec8b76f0f713fc8eb55ff407edce9b413fd Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Tue, 8 Jul 2025 16:15:18 -0500
Subject: [PATCH 06/31] revert foxtrot change

---
 VERSION | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/VERSION b/VERSION
index 452820224..c3344819a 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-2.4.0-foxtrot
\ No newline at end of file
+2.4.170
\ No newline at end of file

From 47bbc9987e509d20d913163b66d91b48034878b3 Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Tue, 8 Jul 2025 16:39:48 -0500
Subject: [PATCH 07/31] elastic agent upgrade prereq

---
 salt/manager/tools/sbin/soup | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup
index f361da688..0c6ebfd26 100755
--- a/salt/manager/tools/sbin/soup
+++ b/salt/manager/tools/sbin/soup
@@ -802,9 +802,6 @@ up_to_2.4.130() {
   # Remove any old Elastic Defend config files
   rm -f /opt/so/conf/elastic-fleet/integrations/endpoints-initial/elastic-defend-endpoints.json
 
-  # Elastic Update for this release, so download Elastic Agent files
-  determine_elastic_agent_upgrade
-
   # Ensure override exists to allow nmcli access to other devices
   touch /etc/NetworkManager/conf.d/10-globally-managed-devices.conf
 
@@ -847,6 +844,9 @@ up_to_2.4.170() {
     touch /opt/so/saltstack/local/pillar/$state/adv_$state.sls /opt/so/saltstack/local/pillar/$state/soc_$state.sls
   done
 
+  # Elastic Update for this release, so download Elastic Agent files
+  determine_elastic_agent_upgrade
+
   INSTALLEDVERSION=2.4.170
 }
 

From 76ab0eac03ce8e2fc1f726e5e561c1425edf89bd Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Tue, 8 Jul 2025 16:45:27 -0500
Subject: [PATCH 08/31] foxtrot

Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com>
---
 VERSION | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/VERSION b/VERSION
index c3344819a..7d52aac7f 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-2.4.170
\ No newline at end of file
+2.4.0-foxtrot

From b9d813cef2b67325e7aa253e66b06da33a111191 Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Tue, 8 Jul 2025 18:26:46 -0500
Subject: [PATCH 09/31] typo

---
 .../tools/sbin_jinja/so-elasticsearch-templates-load            | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load
index ba5767255..4ac1b4d5f 100755
--- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load
+++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load
@@ -136,7 +136,7 @@ if [ ! -f $STATE_FILE_SUCCESS ]; then
       TEMPLATE=${i::-14}
       COMPONENT_PATTERN=${TEMPLATE:3}
       MATCH=$(echo "$TEMPLATE" | grep -E "^so-logs-|^so-metrics" | grep -vE "detections|osquery")
-      if [[ -n "$MATCH" && ! "$COMPONENT_LIST" =~ "$COMPONENT_PATTERN" && ! "$COMPONENT_PATTERN" =~ *\.generic|logs-winlog\.winlog ]]; then
+      if [[ -n "$MATCH" && ! "$COMPONENT_LIST" =~ "$COMPONENT_PATTERN" && ! "$COMPONENT_PATTERN" =~ \.generic|logs-winlog\.winlog ]]; then
         load_failures=$((load_failures+1))
         echo "Component template does not exist for $COMPONENT_PATTERN. The index template will not be loaded. Load failures: $load_failures"
       else

From 07a22a0b4b22ad8f1eaee2d0647c0c12a4a9184c Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Tue, 8 Jul 2025 18:32:14 -0500
Subject: [PATCH 10/31] version

---
 VERSION | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/VERSION b/VERSION
index 7d52aac7f..c3344819a 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-2.4.0-foxtrot
+2.4.170
\ No newline at end of file

From f1cbe23f575ab80c7952212b63269c82291e9662 Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Tue, 8 Jul 2025 21:17:57 -0500
Subject: [PATCH 11/31] update default kibana space

---
 salt/kibana/tools/sbin_jinja/so-kibana-space-defaults | 2 +-
 salt/manager/tools/sbin/soup                          | 6 ++++++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/salt/kibana/tools/sbin_jinja/so-kibana-space-defaults b/salt/kibana/tools/sbin_jinja/so-kibana-space-defaults
index a22aba066..cbd16a2de 100755
--- a/salt/kibana/tools/sbin_jinja/so-kibana-space-defaults
+++ b/salt/kibana/tools/sbin_jinja/so-kibana-space-defaults
@@ -13,6 +13,6 @@ echo "Setting up default Space:"
 {% if HIGHLANDER %}
 curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["enterpriseSearch"]} ' >> /opt/so/log/kibana/misc.log
 {% else %}
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["ml","enterpriseSearch","logs","infrastructure","apm","uptime","monitoring","stackAlerts","actions","securitySolutionCasesV2","inventory","dataQuality","actions"]} ' >> /opt/so/log/kibana/misc.log
+curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["ml","enterpriseSearch","logs","infrastructure","apm","uptime","monitoring","stackAlerts","actions","securitySolutionCasesV3","inventory","dataQuality","searchSynonyms","enterpriseSearchApplications","enterpriseSearchAnalytics","securitySolutionTimeline","securitySolutionNotes","entityManager"]} ' >> /opt/so/log/kibana/misc.log
 {% endif %}
 echo
diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup
index 0c6ebfd26..e37ccbfda 100755
--- a/salt/manager/tools/sbin/soup
+++ b/salt/manager/tools/sbin/soup
@@ -590,6 +590,12 @@ post_to_2.4.160() {
 post_to_2.4.170() {
   echo "Regenerating Elastic Agent Installers"
   /sbin/so-elastic-agent-gen-installers
+
+  # Update kibana default space
+  salt-call state.apply kibana.config queue=True
+  echo "Updating Kibana default space"
+  /usr/sbin/so-kibana-space-defaults
+
   POSTVERSION=2.4.170
 }
 

From 3fb703cd22797e8a641627de65377540821ba258 Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Wed, 9 Jul 2025 11:59:25 -0500
Subject: [PATCH 12/31] check if generic template exists in installed component
 templates before defaulting to logs-filestream.generic@package

---
 salt/elasticfleet/integration-defaults.map.jinja             | 5 +++--
 .../sbin_jinja/so-elastic-fleet-optional-integrations-load   | 2 +-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/salt/elasticfleet/integration-defaults.map.jinja b/salt/elasticfleet/integration-defaults.map.jinja
index 62e37b229..d385c15aa 100644
--- a/salt/elasticfleet/integration-defaults.map.jinja
+++ b/salt/elasticfleet/integration-defaults.map.jinja
@@ -4,7 +4,7 @@
 
 
 {% import_json '/opt/so/state/esfleet_package_components.json' as ADDON_PACKAGE_COMPONENTS %}
-{% import_json '/opt/so/state/esfleet_package_components.json' as INSTALLED_COMPONENT_TEMPLATES %}
+{% import_json '/opt/so/state/esfleet_component_templates.json' as INSTALLED_COMPONENT_TEMPLATES %}
 {% import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %}
 
 {% set CORE_ESFLEET_PACKAGES = ELASTICFLEETDEFAULTS.get('elasticfleet', {}).get('packages', {}) %}
@@ -83,7 +83,8 @@
 {#           pillar overrides/merge expects the key names to follow the naming in elasticsearch/defaults.yaml eg. so-logs-1password_x_item_usages . The _x_ is replaced later on in elasticsearch/template.map.jinja #}
 {%           set integration_key = "so-" ~ integration_type ~ component_name_x %}
 
-{%         if ".generic" in component_name and component_name not in INSTALLED_COMPONENT_TEMPLATES %}
+{#         if its a .generic template make sure that a .generic@package for the integration exists. Else default to logs-filestream.generic@package #}
+{%         if ".generic" in component_name and integration_type ~ component_name ~ "@package" not in INSTALLED_COMPONENT_TEMPLATES %}
 {#           these generic templates by default are directed to index_pattern of 'logs-generic-*', overwrite that here to point to eg gcp_pubsub.generic-* #}
 {%           set index_pattern = integration_type ~ component_name ~ "-*" %}
 {#           includes use of .generic component template, but it doesn't exist in installed component templates. Redirect it to filestream.generic@package #}
diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load
index df01608f5..e877a9e37 100644
--- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load
+++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load
@@ -19,7 +19,7 @@ BULK_INSTALL_PACKAGE_LIST=/tmp/esfleet_bulk_install.json
 BULK_INSTALL_PACKAGE_TMP=/tmp/esfleet_bulk_install_tmp.json
 BULK_INSTALL_OUTPUT=/opt/so/state/esfleet_bulk_install_results.json
 PACKAGE_COMPONENTS=/opt/so/state/esfleet_package_components.json
-COMPONENT_TEMPLATES=/opt/so/state/esfleet_component_templates.txt
+COMPONENT_TEMPLATES=/opt/so/state/esfleet_component_templates.json
 
 PENDING_UPDATE=false
 

From a03764d95678625a29b682e21b20ef43da1eeadb Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Wed, 9 Jul 2025 12:34:53 -0500
Subject: [PATCH 13/31] additional weird integration

---
 salt/elasticfleet/integration-defaults.map.jinja | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/salt/elasticfleet/integration-defaults.map.jinja b/salt/elasticfleet/integration-defaults.map.jinja
index d385c15aa..182dc95a3 100644
--- a/salt/elasticfleet/integration-defaults.map.jinja
+++ b/salt/elasticfleet/integration-defaults.map.jinja
@@ -46,7 +46,10 @@
     'synthetics.browser_screenshot': 'synthetics-browser.screenshot',
     'synthetics.http': 'synthetics-http',
     'synthetics.icmp': 'synthetics-icmp',
-    'synthetics.tcp': 'synthetics-tcp'
+    'synthetics.tcp': 'synthetics-tcp',
+    'swimlane.swimlane_api': 'swimlane.api',
+    'swimlane.tenant_api': 'swimlane.tenant',
+    'swimlane.turbine_api': 'turbine.api'
    } %}
 
 {% for pkg in ADDON_PACKAGE_COMPONENTS %}

From 33e2d18aa7094272b59a50b1af9d1729381b1279 Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Wed, 9 Jul 2025 13:59:01 -0500
Subject: [PATCH 14/31] endpoint policy update

---
 .../elastic-defend-endpoints.json             | 64 ++++++++++---------
 1 file changed, 33 insertions(+), 31 deletions(-)

diff --git a/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json b/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json
index 87870c7bc..ee55551e3 100644
--- a/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json
+++ b/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json
@@ -1,32 +1,34 @@
 {
-	"name": "elastic-defend-endpoints",
-	"namespace": "default",
-	"description": "",
-	"package": {
-	  "name": "endpoint",
-	  "title": "Elastic Defend",
-	  "version": "8.17.0",
-	  "requires_root": true
-	},
-	"enabled": true,
-	"policy_id": "endpoints-initial",
-	"vars": {},
-	"inputs": [
-	  {
-		"type": "endpoint",
-		"enabled": true,
-		"config": {
-		  "integration_config": {
-			"value": {
-			  "type": "endpoint",
-			  "endpointConfig": {
-				"preset": "DataCollection"
-			  }
-			}
-		  }
-		},
-		"streams": []
-	  }
-	]
-  }
-  
\ No newline at end of file
+  "name": "elastic-defend-endpoints",
+  "namespace": "default",
+  "description": "",
+  "package": {
+    "name": "endpoint",
+    "title": "Elastic Defend",
+    "version": "8.18.1",
+    "requires_root": true
+  },
+  "enabled": true,
+  "policy_ids": [
+    "endpoints-initial"
+  ],
+  "vars": {},
+  "inputs": [
+    {
+      "type": "ENDPOINT_INTEGRATION_CONFIG",
+      "enabled": true,
+      "config": {
+        "_config": {
+          "value": {
+            "type": "endpoint",
+            "endpointConfig": {
+              "preset": "DataCollection"
+            }
+          }
+        }
+      },
+      "streams": []
+    }
+  ],
+  "force": false
+}
\ No newline at end of file

From fa6f4100dd350b10cdb98314d48343d46e50990c Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Wed, 9 Jul 2025 14:48:15 -0500
Subject: [PATCH 15/31] ensure elasticsearch is up

---
 .../so-elastic-fleet-optional-integrations-load           | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load
index e877a9e37..8a2c388b1 100644
--- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load
+++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load
@@ -156,9 +156,11 @@ if [[ -f $STATE_FILE_SUCCESS  ]]; then
         # Write out file for generating index/component/ilm templates
         latest_installed_package_list=$(elastic_fleet_installed_packages)
         echo $latest_installed_package_list | jq '[.items[] | {name: .name, es_index_patterns: .dataStreams}]' > $PACKAGE_COMPONENTS
-        # Refresh installed component template list
-        latest_component_templates_list=$(so-elasticsearch-query _component_template | jq '.component_templates[] | .name' | jq -s '.')
-        echo $latest_component_templates_list > $COMPONENT_TEMPLATES
+        if retry 3 1 "so-elasticsearch-query / --fail --output /dev/null"; then
+            # Refresh installed component template list
+            latest_component_templates_list=$(so-elasticsearch-query _component_template | jq '.component_templates[] | .name' | jq -s '.')
+            echo $latest_component_templates_list > $COMPONENT_TEMPLATES
+        fi
 
     else
         # This is the installation of add-on integrations and upgrade of existing integrations. Exiting without error, next highstate will attempt to re-run.

From 3b2942651e9890726d1a040086b69d519790a736 Mon Sep 17 00:00:00 2001
From: Jorge Reyes <94730068+reyesj2@users.noreply.github.com>
Date: Wed, 9 Jul 2025 15:14:24 -0500
Subject: [PATCH 16/31] Update
 salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json

---
 .../integrations/elastic-defend/elastic-defend-endpoints.json   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json b/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json
index ee55551e3..27162669d 100644
--- a/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json
+++ b/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json
@@ -30,5 +30,5 @@
       "streams": []
     }
   ],
-  "force": false
+}
 }
\ No newline at end of file

From d846fe55e14d93666dfd7e13ff1411863d2b1abb Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Wed, 9 Jul 2025 15:40:36 -0500
Subject: [PATCH 17/31] typos

---
 .../integrations/elastic-defend/elastic-defend-endpoints.json  | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json b/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json
index 27162669d..fb9069e83 100644
--- a/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json
+++ b/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json
@@ -29,6 +29,5 @@
       },
       "streams": []
     }
-  ],
-}
+  ]
 }
\ No newline at end of file

From 317d7dea7d7c5561a42662cbb546cd4c7424c6f9 Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Wed, 9 Jul 2025 17:25:36 -0500
Subject: [PATCH 18/31] check required files exist before loading map file

Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com>
---
 salt/elasticsearch/template.map.jinja    | 2 +-
 salt/manager/managed_soc_annotations.sls | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/salt/elasticsearch/template.map.jinja b/salt/elasticsearch/template.map.jinja
index aa90cb81b..414d8a6b4 100644
--- a/salt/elasticsearch/template.map.jinja
+++ b/salt/elasticsearch/template.map.jinja
@@ -15,7 +15,7 @@
 {% set ES_INDEX_SETTINGS_ORIG = ELASTICSEARCHDEFAULTS.elasticsearch.index_settings %}
 
 {# start generation of integration default index_settings #}
-{% if salt['file.file_exists']('/opt/so/state/esfleet_package_components.json') %}
+{% if salt['file.file_exists']('/opt/so/state/esfleet_package_components.json') and salt['file.file_exists']('/opt/so/state/esfleet_component_templates.json') %}
 {%   set check_package_components = salt['file.stats']('/opt/so/state/esfleet_package_components.json') %}
 {%   if check_package_components.size > 1 %}
 {%    from 'elasticfleet/integration-defaults.map.jinja' import ADDON_INTEGRATION_DEFAULTS %}
diff --git a/salt/manager/managed_soc_annotations.sls b/salt/manager/managed_soc_annotations.sls
index 9ef0b1ea1..d8f175df6 100644
--- a/salt/manager/managed_soc_annotations.sls
+++ b/salt/manager/managed_soc_annotations.sls
@@ -5,7 +5,7 @@
 
 {# Managed elasticsearch/soc_elasticsearch.yaml file for adding integration configuration items to UI #}
 {% set managed_integrations = salt['pillar.get']('elasticsearch:managed_integrations', []) %}
-{% if managed_integrations and salt['file.file_exists']('/opt/so/state/esfleet_package_components.json') %}
+{% if managed_integrations and salt['file.file_exists']('/opt/so/state/esfleet_package_components.json') and salt['file.file_exists']('/opt/so/state/esfleet_component_templates.json') %}
 {%   from 'elasticfleet/integration-defaults.map.jinja' import ADDON_INTEGRATION_DEFAULTS %}
 {%   set addon_integration_keys = ADDON_INTEGRATION_DEFAULTS.keys() %}
 {%   set matched_integration_names = [] %}

From bf8da6060555c57de61b616493f4b8d305c53b53 Mon Sep 17 00:00:00 2001
From: Jason Ertel <jason.ertel@securityonionsolutions.com>
Date: Thu, 10 Jul 2025 07:47:53 -0400
Subject: [PATCH 19/31] exclude component updates indexes with error in the
 name

---
 salt/common/tools/sbin/so-log-check | 1 +
 1 file changed, 1 insertion(+)

diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check
index 96881aa56..c84fc7803 100755
--- a/salt/common/tools/sbin/so-log-check
+++ b/salt/common/tools/sbin/so-log-check
@@ -158,6 +158,7 @@ if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|adding index lifecycle policy" # false positive (elasticsearch policy names contain 'error') 
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|adding ingest pipeline"       # false positive (elasticsearch ingest pipeline names contain 'error') 
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|updating index template"      # false positive (elasticsearch index or template names contain 'error') 
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|updating component template"  # false positive (elasticsearch index or template names contain 'error') 
 fi
 
 if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then

From 95ba327eb3e9eab2b031cf80c49658cedb0b5b79 Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Thu, 10 Jul 2025 11:08:46 -0500
Subject: [PATCH 20/31] cribl metrics template rename

---
 salt/elasticfleet/integration-defaults.map.jinja | 1 +
 1 file changed, 1 insertion(+)

diff --git a/salt/elasticfleet/integration-defaults.map.jinja b/salt/elasticfleet/integration-defaults.map.jinja
index 182dc95a3..500a9e63c 100644
--- a/salt/elasticfleet/integration-defaults.map.jinja
+++ b/salt/elasticfleet/integration-defaults.map.jinja
@@ -15,6 +15,7 @@
     'awsfirehose.logs': 'awsfirehose',
     'awsfirehose.metrics': 'aws.cloudwatch',
     'cribl.logs': 'cribl',
+    'cribl.metrics': 'cribl',
     'sentinel_one_cloud_funnel.logins': 'sentinel_one_cloud_funnel.login',
     'azure_application_insights.app_insights': 'azure.app_insights',
     'azure_application_insights.app_state': 'azure.app_state',

From a4e8e7ea535251d9aeb826a5fd97c9ef18fe977e Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Thu, 10 Jul 2025 13:12:26 -0500
Subject: [PATCH 21/31] update syslog-tcp-514 policy

---
 .../integrations/grid-nodes_general/syslog-tcp-514.json      | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/syslog-tcp-514.json b/salt/elasticfleet/files/integrations/grid-nodes_general/syslog-tcp-514.json
index 4088f5a87..f284ede06 100644
--- a/salt/elasticfleet/files/integrations/grid-nodes_general/syslog-tcp-514.json
+++ b/salt/elasticfleet/files/integrations/grid-nodes_general/syslog-tcp-514.json
@@ -11,7 +11,7 @@
     "tcp-tcp": {
       "enabled": true,
       "streams": {
-        "tcp.generic": {
+        "tcp.tcp": {
           "enabled": true,
           "vars": {
             "listen_address": "0.0.0.0",
@@ -23,7 +23,8 @@
               "syslog"
             ],
             "syslog_options": "field: message\n#format: auto\n#timezone: Local",
-            "ssl": ""
+            "ssl": "",
+            "custom": ""
           }
         }
       }

From 8a57b79b779f60fa04b22cdd8226bbd1f1eb07c7 Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Thu, 10 Jul 2025 15:52:59 -0500
Subject: [PATCH 22/31] make package installs go in groups of 25 or less

---
 ...so-elastic-fleet-optional-integrations-load | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load
index 8a2c388b1..886bbf75c 100644
--- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load
+++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load
@@ -148,8 +148,22 @@ if [[ -f $STATE_FILE_SUCCESS  ]]; then
         done <<< "$(jq -c '.packages[]' "$INSTALLED_PACKAGE_LIST")"
 
         if [ "$PENDING_UPDATE" = true ]; then
-            # Run bulk install of packages
-            elastic_fleet_bulk_package_install $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_OUTPUT
+            # Run chunked install of packages
+            echo "" > $BULK_INSTALL_OUTPUT
+            pkg_group=1
+            pkg_filename="${BULK_INSTALL_PACKAGE_LIST%.json}"
+
+            jq -c '.packages | _nwise(25)' $BULK_INSTALL_PACKAGE_LIST | while read -r line; do
+                echo "$line" | jq '{ "packages": . }' > "${pkg_filename}_${pkg_group}.json"
+                pkg_group=$((pkg_group + 1))
+            done
+
+            for file in "${pkg_filename}_"*.json; do
+                [ -e "$file" ] || continue
+                elastic_fleet_bulk_package_install $file >> $BULK_INSTALL_OUTPUT
+            done
+            # cleanup any temp files for chunked package install
+            rm -f ${pkg_filename}_*.json $BULK_INSTALL_PACKAGE_LIST
         else
             echo "Elastic integrations don't appear to need installation/updating..."
         fi

From 10e3b32fedd1bdb3cf87b82b329692aa2021c4d3 Mon Sep 17 00:00:00 2001
From: Jason Ertel <jason.ertel@securityonionsolutions.com>
Date: Fri, 11 Jul 2025 11:29:16 -0400
Subject: [PATCH 23/31] fix typo

---
 salt/soc/soc_soc.yaml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml
index 58560e89e..00342b4b8 100644
--- a/salt/soc/soc_soc.yaml
+++ b/salt/soc/soc_soc.yaml
@@ -589,6 +589,10 @@ soc:
           global: True
           advanced: True
           forcedType: "[]{}"
+        exportNodeId:
+          description: The node ID on which export jobs will be executed.
+          global: True
+          advanced: True
         hunt: &appSettings
           groupItemsPerPage:
             description: Default number of aggregations to show per page. Larger values consume more vertical area in the SOC UI.
@@ -712,7 +716,7 @@ soc:
                 global: True
             status:
               labels:
-                description: List of available case statuses. Some statuses have specifial characteristics and related functionality built into SOC.
+                description: List of available case statuses. Note that some default statuses have special characteristics and related functionality built into SOC.
                 global: True
               customEnabled:
                 description: Set to true to allow users add their own case statuses directly in the SOC UI.

From 9b125fbe531dbbf76bffb68883da934cf4ef2508 Mon Sep 17 00:00:00 2001
From: Jason Ertel <jason.ertel@securityonionsolutions.com>
Date: Fri, 11 Jul 2025 11:30:01 -0400
Subject: [PATCH 24/31] fix typo

---
 salt/soc/soc_soc.yaml | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml
index 00342b4b8..da3549039 100644
--- a/salt/soc/soc_soc.yaml
+++ b/salt/soc/soc_soc.yaml
@@ -589,10 +589,6 @@ soc:
           global: True
           advanced: True
           forcedType: "[]{}"
-        exportNodeId:
-          description: The node ID on which export jobs will be executed.
-          global: True
-          advanced: True
         hunt: &appSettings
           groupItemsPerPage:
             description: Default number of aggregations to show per page. Larger values consume more vertical area in the SOC UI.

From 415f456661fd24b846c68159000c882993d4c9de Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Sat, 12 Jul 2025 08:30:04 -0500
Subject: [PATCH 25/31] ignore composable templates with error in the name

---
 salt/common/tools/sbin/so-log-check | 1 +
 1 file changed, 1 insertion(+)

diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check
index c84fc7803..72ece1919 100755
--- a/salt/common/tools/sbin/so-log-check
+++ b/salt/common/tools/sbin/so-log-check
@@ -159,6 +159,7 @@ if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|adding ingest pipeline"       # false positive (elasticsearch ingest pipeline names contain 'error') 
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|updating index template"      # false positive (elasticsearch index or template names contain 'error') 
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|updating component template"  # false positive (elasticsearch index or template names contain 'error') 
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|upgrading composable template" # false positive (elasticsearch composable template names contain 'error')
 fi
 
 if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then

From f8108e93d5a9a99c16c471d9132d1a225a9f4f2c Mon Sep 17 00:00:00 2001
From: Doug Burks <doug.burks@securityonionsolutions.com>
Date: Mon, 14 Jul 2025 12:04:46 -0400
Subject: [PATCH 26/31] FEATURE: Add SOC default fields for iptables logs
 #14836

---
 salt/soc/defaults.yaml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml
index cb12671f8..35eb22ab0 100644
--- a/salt/soc/defaults.yaml
+++ b/salt/soc/defaults.yaml
@@ -1336,6 +1336,13 @@ soc:
         - soc.fields.statusCode
         - event.action
         - soc.fields.error
+      ':iptables:':
+        - soc_timestamp
+        - source.ip
+        - source.port
+        - destination.ip
+        - destination.port
+        - message
     server:
       bindAddress: 0.0.0.0:9822
       baseUrl: /

From 10bf3e8fab0681d0f9f2f11fa7a23666f7fc5fe9 Mon Sep 17 00:00:00 2001
From: Doug Burks <doug.burks@securityonionsolutions.com>
Date: Mon, 14 Jul 2025 12:07:02 -0400
Subject: [PATCH 27/31] FEATURE: Add SOC default fields for CEF logs #14837

---
 salt/soc/defaults.yaml | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml
index 35eb22ab0..e84a5b017 100644
--- a/salt/soc/defaults.yaml
+++ b/salt/soc/defaults.yaml
@@ -1343,6 +1343,14 @@ soc:
         - destination.ip
         - destination.port
         - message
+      ':cef:':
+        - soc_timestamp
+        - cef.device.event_class_id
+        - cef.device.vendor
+        - cef.device.product
+        - cef.device.version
+        - log.source.address
+        - message
     server:
       bindAddress: 0.0.0.0:9822
       baseUrl: /

From ab9d03bc2e7cd4b38f08e596ab14e4ad205007d2 Mon Sep 17 00:00:00 2001
From: Doug Burks <doug.burks@securityonionsolutions.com>
Date: Mon, 14 Jul 2025 12:21:08 -0400
Subject: [PATCH 28/31] FEATURE: Add SOC Dashboards for UniFi logs #14838

---
 salt/soc/defaults.yaml | 30 +++++++++++++++++++++---------
 1 file changed, 21 insertions(+), 9 deletions(-)

diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml
index e84a5b017..23c966bb4 100644
--- a/salt/soc/defaults.yaml
+++ b/salt/soc/defaults.yaml
@@ -2142,15 +2142,6 @@ soc:
             - name: ICS S7
               description: S7 (Siemens) network metadata
               query: 'tags:s7* | groupby event.dataset | groupby -sankey event.dataset source.ip | groupby source.ip | groupby -sankey source.ip destination.ip | groupby destination.ip | groupby destination.port'
-            - name: NetFlow
-              description: NetFlow records
-              query: 'event.module:netflow | groupby source.ip | groupby -sankey source.ip destination.ip | groupby destination.ip | groupby destination.port | groupby network.type | groupby network.transport | groupby network.direction | groupby netflow.type | groupby netflow.exporter.version  | groupby observer.ip | groupby source.as.organization.name | groupby source.geo.country_name | groupby destination.as.organization.name | groupby destination.geo.country_name'
-            - name: Firewall
-              description: Firewall logs
-              query: 'observer.type:firewall | groupby event.action | groupby -sankey event.action observer.ingress.interface.name | groupby observer.ingress.interface.name | groupby network.type | groupby network.transport | groupby source.ip | groupby destination.ip | groupby destination.port'
-            - name: Firewall Auth
-              description: Firewall authentication logs
-              query: 'observer.type:firewall AND event.category:authentication | groupby user.name | groupby -sankey user.name source.ip | groupby source.ip | table soc_timestamp user.name source.ip message'
             - name: VLAN
               description: VLAN (Virtual Local Area Network) tagged logs
               query: '* AND _exists_:network.vlan.id | groupby network.vlan.id | groupby -sankey network.vlan.id source.ip | groupby source.ip | groupby destination.ip | groupby destination.port | groupby event.dataset | groupby event.module | groupby observer.name | groupby source.geo.country_name | groupby destination.geo.country_name'
@@ -2166,6 +2157,27 @@ soc:
             - name: GeoIP - Source Organizations
               description: GeoIP tagged logs visualized by source organizations
               query: '* AND _exists_:source_geo.organization_name | groupby source_geo.organization_name | groupby source.ip | groupby -sankey source.ip destination.ip | groupby destination.ip | groupby destination.port | groupby source.geo.country_name | groupby event.dataset | groupby event.module'
+            - name: NetFlow
+              description: NetFlow records
+              query: 'event.module:netflow | groupby source.ip | groupby -sankey source.ip destination.ip | groupby destination.ip | groupby destination.port | groupby network.type | groupby network.transport | groupby network.direction | groupby netflow.type | groupby netflow.exporter.version  | groupby observer.ip | groupby source.as.organization.name | groupby source.geo.country_name | groupby destination.as.organization.name | groupby destination.geo.country_name'
+            - name: Firewall - pfSense/OPNsense
+              description: pfSense/OPNsense firewall logs
+              query: 'observer.type:firewall | groupby event.action | groupby -sankey event.action observer.ingress.interface.name | groupby observer.ingress.interface.name | groupby network.type | groupby network.transport | groupby source.ip | groupby destination.ip | groupby destination.port'
+            - name: Firewall - pfSense/OPNsense Auth
+              description: pfSense/OPNsense firewall authentication logs
+              query: 'observer.type:firewall AND event.category:authentication | groupby user.name | groupby -sankey user.name source.ip | groupby source.ip | table soc_timestamp user.name source.ip message'
+            - name: Firewall - UniFi Firewall Overview
+              description: All network traffic logged by UniFi firewall
+              query: 'event.module:iptables AND event.type:connection | groupby source.ip | groupby -sankey source.ip destination.ip | groupby destination.ip | groupby -sankey destination.ip destination.port | groupby destination.port'
+            - name: Firewall - UniFi Firewall Blocks
+              description: Network traffic blocked by UniFi firewall
+              query: 'event.module:iptables AND event.type:connection AND (message:iptables-dropped OR message:block) | groupby source.ip | groupby -sankey source.ip destination.ip | groupby destination.ip | groupby -sankey destination.ip destination.port | groupby destination.port'
+            - name: Firewall - UniFi Firewall Allows
+              description: Network traffic allowed by UniFi firewall
+              query: 'event.module:iptables AND event.type:connection AND NOT (message:iptables-dropped OR message:block) | groupby source.ip | groupby -sankey source.ip destination.ip | groupby destination.ip | groupby -sankey destination.ip destination.port | groupby destination.port'
+            - name: Firewall - UniFi Auth
+              description: UniFi authentication logs
+              query: 'event.module:cef | groupby cef.device.event_class_id | groupby -sankey cef.device.event_class_id cef.device.vendor | groupby cef.device.vendor | groupby cef.device.product | groupby cef.device.version | groupby log.source.address'
             - name: Kismet - WiFi Devices
               description: WiFi devices seen by Kismet sensors
               query: 'event.module: kismet | groupby network.wireless.ssid | groupby device.manufacturer | groupby -pie device.manufacturer | groupby event.dataset'

From 4f8bd16910050c973d2dfc8e65fb1c599516e3fa Mon Sep 17 00:00:00 2001
From: Doug Burks <doug.burks@securityonionsolutions.com>
Date: Mon, 14 Jul 2025 15:37:10 -0400
Subject: [PATCH 29/31] FEATURE: Add SOC Dashboards for CEF, iptables, and
 UniFi logs #14838

---
 salt/soc/defaults.yaml | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml
index 23c966bb4..0c5967753 100644
--- a/salt/soc/defaults.yaml
+++ b/salt/soc/defaults.yaml
@@ -2166,6 +2166,9 @@ soc:
             - name: Firewall - pfSense/OPNsense Auth
               description: pfSense/OPNsense firewall authentication logs
               query: 'observer.type:firewall AND event.category:authentication | groupby user.name | groupby -sankey user.name source.ip | groupby source.ip | table soc_timestamp user.name source.ip message'
+            - name: Firewall - iptables
+              description: All network traffic logged by Elastic integration for iptables
+              query: 'event.module:iptables AND event.type:connection | groupby source.ip | groupby -sankey source.ip destination.ip | groupby destination.ip | groupby -sankey destination.ip destination.port | groupby destination.port'
             - name: Firewall - UniFi Firewall Overview
               description: All network traffic logged by UniFi firewall
               query: 'event.module:iptables AND event.type:connection | groupby source.ip | groupby -sankey source.ip destination.ip | groupby destination.ip | groupby -sankey destination.ip destination.port | groupby destination.port'
@@ -2175,8 +2178,11 @@ soc:
             - name: Firewall - UniFi Firewall Allows
               description: Network traffic allowed by UniFi firewall
               query: 'event.module:iptables AND event.type:connection AND NOT (message:iptables-dropped OR message:block) | groupby source.ip | groupby -sankey source.ip destination.ip | groupby destination.ip | groupby -sankey destination.ip destination.port | groupby destination.port'
-            - name: Firewall - UniFi Auth
-              description: UniFi authentication logs
+            - name: Firewall - UniFi System
+              description: UniFi system logs
+              query: 'event.module:cef | groupby cef.device.event_class_id | groupby -sankey cef.device.event_class_id cef.device.vendor | groupby cef.device.vendor | groupby cef.device.product | groupby cef.device.version | groupby log.source.address'
+            - name: CEF
+              description: Logs handled by the Elastic integration for CEF
               query: 'event.module:cef | groupby cef.device.event_class_id | groupby -sankey cef.device.event_class_id cef.device.vendor | groupby cef.device.vendor | groupby cef.device.product | groupby cef.device.version | groupby log.source.address'
             - name: Kismet - WiFi Devices
               description: WiFi devices seen by Kismet sensors
@@ -2184,9 +2190,6 @@ soc:
             - name: SOC Detections - Runtime Status
               description: Runtime Status of Detections
               query: 'event.dataset:soc.detections | groupby  soc.detection_type soc.error_type | groupby soc.error_analysis | groupby soc.rule.name | groupby soc.error_message'
-
-
-
         job:
         alerts:
           advanced: false

From 81d2c5286727e7ab7033213d7a307aeed1f28ac2 Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Mon, 14 Jul 2025 16:08:11 -0500
Subject: [PATCH 30/31] kibana listingLimit

---
 salt/kibana/files/config_saved_objects.ndjson.jinja | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/kibana/files/config_saved_objects.ndjson.jinja b/salt/kibana/files/config_saved_objects.ndjson.jinja
index 4902a1445..b1c2f0fb7 100644
--- a/salt/kibana/files/config_saved_objects.ndjson.jinja
+++ b/salt/kibana/files/config_saved_objects.ndjson.jinja
@@ -1,3 +1,3 @@
 {% import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS -%}
 
-{"attributes": {"buildNum": 39457,"defaultIndex": "logs-*","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n  \"from\": \"now-24h\",\n  \"to\": \"now\"\n}"},"coreMigrationVersion": "{{ ELASTICSEARCHDEFAULTS.elasticsearch.version }}","id": "{{ ELASTICSEARCHDEFAULTS.elasticsearch.version }}","references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="}
+{"attributes": {"buildNum": 39457,"defaultIndex": "logs-*","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"savedObjects:listingLimit":1500,"theme:darkMode": true,"timepicker:timeDefaults": "{\n  \"from\": \"now-24h\",\n  \"to\": \"now\"\n}"},"coreMigrationVersion": "{{ ELASTICSEARCHDEFAULTS.elasticsearch.version }}","id": "{{ ELASTICSEARCHDEFAULTS.elasticsearch.version }}","references": [],"type": "config","version": "WzI5NzUsMl0="}

From 6bb6c24641d26f96585050df531f0dc8bdcf9d3a Mon Sep 17 00:00:00 2001
From: Doug Burks <doug.burks@securityonionsolutions.com>
Date: Wed, 16 Jul 2025 07:20:39 -0400
Subject: [PATCH 31/31] Simplify UniFi dashboards #14838

---
 salt/soc/defaults.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml
index 0c5967753..ad7d51f68 100644
--- a/salt/soc/defaults.yaml
+++ b/salt/soc/defaults.yaml
@@ -2174,10 +2174,10 @@ soc:
               query: 'event.module:iptables AND event.type:connection | groupby source.ip | groupby -sankey source.ip destination.ip | groupby destination.ip | groupby -sankey destination.ip destination.port | groupby destination.port'
             - name: Firewall - UniFi Firewall Blocks
               description: Network traffic blocked by UniFi firewall
-              query: 'event.module:iptables AND event.type:connection AND (message:iptables-dropped OR message:block) | groupby source.ip | groupby -sankey source.ip destination.ip | groupby destination.ip | groupby -sankey destination.ip destination.port | groupby destination.port'
+              query: 'event.module:iptables AND event.type:connection AND message:block | groupby source.ip | groupby -sankey source.ip destination.ip | groupby destination.ip | groupby -sankey destination.ip destination.port | groupby destination.port'
             - name: Firewall - UniFi Firewall Allows
               description: Network traffic allowed by UniFi firewall
-              query: 'event.module:iptables AND event.type:connection AND NOT (message:iptables-dropped OR message:block) | groupby source.ip | groupby -sankey source.ip destination.ip | groupby destination.ip | groupby -sankey destination.ip destination.port | groupby destination.port'
+              query: 'event.module:iptables AND event.type:connection AND NOT message:block | groupby source.ip | groupby -sankey source.ip destination.ip | groupby destination.ip | groupby -sankey destination.ip destination.port | groupby destination.port'
             - name: Firewall - UniFi System
               description: UniFi system logs
               query: 'event.module:cef | groupby cef.device.event_class_id | groupby -sankey cef.device.event_class_id cef.device.vendor | groupby cef.device.vendor | groupby cef.device.product | groupby cef.device.version | groupby log.source.address'