Add Gemma configuration to defaults.yaml

Support sigma playbooks for airgap

salt/common/tools/sbin/so-log-check

@@ -231,7 +231,7 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|from NIC checksum offloading" # zeek reporter.log
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|marked for removal"           # docker container getting recycled
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|tcp 127.0.0.1:6791: bind: address already in use" # so-elastic-fleet agent restarting. Seen starting w/ 8.18.8 https://github.com/elastic/kibana/issues/201459
-    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|TransformTask\] \[logs-(tychon|aws_billing|microsoft_defender_endpoint|armis|o365_metrics|microsoft_sentinel|snyk|cyera|island_browser).*user so_kibana lacks the required permissions \[(logs|metrics)-\1" # Known issue with integrations starting transform jobs that are explicitly not allowed to start as a system user. This error should not be seen on fresh ES 9.3.3 installs or after SO 3.1.0 with soups addition of check_transform_health_and_reauthorize()
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|TransformTask\] \[logs-.*user so_kibana lacks the required permissions" # Known issue with integrations starting transform jobs that are explicitly not allowed to start as a system user
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|manifest unknown"             # appears in so-dockerregistry log for so-tcpreplay following docker upgrade to 29.2.1-1
 fi
 

salt/common/tools/sbin/so-restart

@@ -35,9 +35,6 @@ case $1 in
     "elastic-fleet"|"elasticfleet")
         docker_check_running "elastic-fleet" "--stop"
         docker rm "so-elastic-fleet" 2> /dev/null
-        # Removing the elastic fleet state directory, so that the next startup re-enrolls with a fresh policy
-        rm -rf /opt/so/conf/elastic-fleet/state
-
         salt-call state.apply elasticfleet queue=True
         ;;
     *)

salt/common/tools/sbin/so-stop

@@ -29,8 +29,6 @@ case $1 in
 	"elasticfleet"|"elastic-fleet")
         docker_check_running "elastic-fleet" "--stop"
         docker rm "so-elastic-fleet" 2> /dev/null
-        # Removing the elastic fleet state directory, so that the next startup re-enrolls with a fresh policy
-        rm -rf /opt/so/conf/elastic-fleet/state
         ;;
     *)
         docker_check_running "$1" "--stop"

salt/common/tools/sbin_jinja/so-salt-minion-check

@@ -25,7 +25,7 @@ LAST_HIGHSTATE_END=$([ -e "/opt/so/log/salt/lasthighstate" ] && date -r /opt/so/
 LAST_HEALTHCHECK_STATE_APPLY=$([ -e "/opt/so/log/salt/state-apply-test" ] && date -r /opt/so/log/salt/state-apply-test +%s || echo 0)
 # SETTING THRESHOLD TO ANYTHING UNDER 600 seconds may cause a lot of salt-minion restarts since the job to touch the file occurs every 5-8 minutes by default
 # THRESHOLD is derived from the salt schedule highstate interval + 1 hour, so the minion-check grace period tracks the schedule automatically.
-THRESHOLD=$(( ({{ SCHEDULEMERGED.highstate_interval_hours }} + 1) * 3600 )) #within how many seconds the file /opt/so/log/salt/state-apply-test must have been touched/modified before the salt minion is restarted
+THRESHOLD=$(( ({{ SCHEDULEMERGED.highstate_interval_minutes }} + 60) * 60 )) #within how many seconds the file /opt/so/log/salt/state-apply-test must have been touched/modified before the salt minion is restarted
 THRESHOLD_DATE=$((LAST_HEALTHCHECK_STATE_APPLY+THRESHOLD))
 
 logCmd() {

salt/elasticfleet/enabled.sls

@@ -11,10 +11,6 @@
 
 {#   This value is generated during node install and stored in minion pillar #}
 {%   set SERVICETOKEN = salt['pillar.get']('elasticfleet:config:server:es_token','') %}
-{#   Prevent Elastic Agent from re-enrolling with a new agent.id everytime the container starts up.
-       - if a fresh enrollment is needed use 'so-stop elasticfleet'
-       #}
-{%   set ENROLLED = salt['file.file_exists']('/opt/so/conf/elastic-fleet/state/fleet.enc') %}
 
 include:
   - ca
@@ -70,7 +66,6 @@ so-elastic-fleet:
       - /etc/pki/elasticfleet-server.crt:/etc/pki/elasticfleet-server.crt:ro
       - /etc/pki/elasticfleet-server.key:/etc/pki/elasticfleet-server.key:ro
       - /etc/pki/tls/certs/intca.crt:/etc/pki/tls/certs/intca.crt:ro
-      - /opt/so/conf/elastic-fleet/state:/usr/share/elastic-agent/state
       - /opt/so/log/elasticfleet:/usr/share/elastic-agent/logs 
      {% if DOCKERMERGED.containers['so-elastic-fleet'].custom_bind_mounts %}
         {% for BIND in DOCKERMERGED.containers['so-elastic-fleet'].custom_bind_mounts %}
@@ -78,7 +73,6 @@ so-elastic-fleet:
         {% endfor %}
       {% endif %}      
     - environment:
-      {% if not ENROLLED %}
       - FLEET_SERVER_ENABLE=true
       - FLEET_URL=https://{{ GLOBALS.hostname }}:8220
       - FLEET_SERVER_ELASTICSEARCH_HOST=https://{{ GLOBALS.manager }}:9200
@@ -88,9 +82,6 @@ so-elastic-fleet:
       - FLEET_SERVER_CERT_KEY=/etc/pki/elasticfleet-server.key
       - FLEET_CA=/etc/pki/tls/certs/intca.crt     
       - FLEET_SERVER_ELASTICSEARCH_CA=/etc/pki/tls/certs/intca.crt
-      {% endif %}
-      - STATE_PATH=/usr/share/elastic-agent/state
-      - CONFIG_PATH=/usr/share/elastic-agent/state
       - LOGS_PATH=logs
       {% if DOCKERMERGED.containers['so-elastic-fleet'].extra_env %}
         {% for XTRAENV in DOCKERMERGED.containers['so-elastic-fleet'].extra_env %}
@@ -109,7 +100,6 @@ so-elastic-fleet:
       - x509: etc_elasticfleet_crt
     - require:
       - file: trusttheca
-      - file: eastatedir
       - x509: etc_elasticfleet_key
       - x509: etc_elasticfleet_crt
 

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade

@@ -17,7 +17,7 @@ if INSTALLED_VERSION=$(elastic_fleet_package_version_check "{{ PACKAGE }}") && L
     if [ "$INSTALLED_VERSION" == "$LATEST_VERSION" ]; then
         echo "{{ PACKAGE }} integration version $INSTALLED_VERSION is already at the reported latest version $LATEST_VERSION, skipping upgrade."
     else
-        echo "Upgrading {{ PACKAGE }} package to version $LATEST_VERSION..."
+        echo "Upgrading {{ PACKAGE }} package from $INSTALLED_VERSION to version $LATEST_VERSION..."
         if ! elastic_fleet_package_install "{{ PACKAGE }}" "$LATEST_VERSION"; then
             PKG_LOAD_FAILURES=$((PKG_LOAD_FAILURES + 1))
             PKG_LOAD_FAILURES_NAMES+=("{{ PACKAGE }}")

salt/elasticsearch/defaults.yaml

@@ -1,5 +1,6 @@
 elasticsearch:
   enabled: false
+  esheap: '600m'
   version: 9.3.7
   index_clean: true
   data_retention_method: DLM
@@ -3453,6 +3454,720 @@ elasticsearch:
               set_priority:
                 priority: 50
             min_age: 30d
+    so-metrics-system_x_core:
+      data_stream_lifecycle:
+        data_retention: 90d
+      index_template:
+        composed_of:
+        - metrics@tsdb-settings
+        - metrics-system.core@package
+        - metrics@custom
+        - system@custom
+        - metrics-system.core@custom
+        - ecs@mappings
+        - so-fleet_integrations.ip_mappings-1
+        - so-fleet_globals-1
+        - so-fleet_agent_id_verification-1
+        data_stream:
+          allow_custom_routing: false
+          hidden: false
+        ignore_missing_component_templates:
+        - metrics@custom
+        - system@custom
+        - metrics-system.core@custom
+        index_patterns:
+        - metrics-system.core-*
+        priority: 501
+        template:
+          settings:
+            index:
+              lifecycle:
+                name: so-metrics-system.core-logs
+              mode: time_series
+              number_of_replicas: 0
+      policy:
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 60d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
+    so-metrics-system_x_cpu:
+      data_stream_lifecycle:
+        data_retention: 90d
+      index_template:
+        composed_of:
+        - metrics@tsdb-settings
+        - metrics-system.cpu@package
+        - metrics@custom
+        - system@custom
+        - metrics-system.cpu@custom
+        - ecs@mappings
+        - so-fleet_integrations.ip_mappings-1
+        - so-fleet_globals-1
+        - so-fleet_agent_id_verification-1
+        data_stream:
+          allow_custom_routing: false
+          hidden: false
+        ignore_missing_component_templates:
+        - metrics@custom
+        - system@custom
+        - metrics-system.cpu@custom
+        index_patterns:
+        - metrics-system.cpu-*
+        priority: 501
+        template:
+          settings:
+            index:
+              lifecycle:
+                name: so-metrics-system.cpu-logs
+              mode: time_series
+              number_of_replicas: 0
+      policy:
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 60d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
+    so-metrics-system_x_diskio:
+      data_stream_lifecycle:
+        data_retention: 90d
+      index_template:
+        composed_of:
+        - metrics@tsdb-settings
+        - metrics-system.diskio@package
+        - metrics@custom
+        - system@custom
+        - metrics-system.diskio@custom
+        - ecs@mappings
+        - so-fleet_integrations.ip_mappings-1
+        - so-fleet_globals-1
+        - so-fleet_agent_id_verification-1
+        data_stream:
+          allow_custom_routing: false
+          hidden: false
+        ignore_missing_component_templates:
+        - metrics@custom
+        - system@custom
+        - metrics-system.diskio@custom
+        index_patterns:
+        - metrics-system.diskio-*
+        priority: 501
+        template:
+          settings:
+            index:
+              lifecycle:
+                name: so-metrics-system.diskio-logs
+              mode: time_series
+              number_of_replicas: 0
+      policy:
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 60d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
+    so-metrics-system_x_filesystem:
+      data_stream_lifecycle:
+        data_retention: 90d
+      index_template:
+        composed_of:
+        - metrics@tsdb-settings
+        - metrics-system.filesystem@package
+        - metrics@custom
+        - system@custom
+        - metrics-system.filesystem@custom
+        - ecs@mappings
+        - so-fleet_integrations.ip_mappings-1
+        - so-fleet_globals-1
+        - so-fleet_agent_id_verification-1
+        data_stream:
+          allow_custom_routing: false
+          hidden: false
+        ignore_missing_component_templates:
+        - metrics@custom
+        - system@custom
+        - metrics-system.filesystem@custom
+        index_patterns:
+        - metrics-system.filesystem-*
+        priority: 501
+        template:
+          settings:
+            index:
+              lifecycle:
+                name: so-metrics-system.filesystem-logs
+              mode: time_series
+              number_of_replicas: 0
+      policy:
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 60d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
+    so-metrics-system_x_fsstat:
+      data_stream_lifecycle:
+        data_retention: 90d
+      index_template:
+        composed_of:
+        - metrics@tsdb-settings
+        - metrics-system.fsstat@package
+        - metrics@custom
+        - system@custom
+        - metrics-system.fsstat@custom
+        - ecs@mappings
+        - so-fleet_integrations.ip_mappings-1
+        - so-fleet_globals-1
+        - so-fleet_agent_id_verification-1
+        data_stream:
+          allow_custom_routing: false
+          hidden: false
+        ignore_missing_component_templates:
+        - metrics@custom
+        - system@custom
+        - metrics-system.fsstat@custom
+        index_patterns:
+        - metrics-system.fsstat-*
+        priority: 501
+        template:
+          settings:
+            index:
+              lifecycle:
+                name: so-metrics-system.fsstat-logs
+              mode: time_series
+              number_of_replicas: 0
+      policy:
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 60d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
+    so-metrics-system_x_load:
+      data_stream_lifecycle:
+        data_retention: 90d
+      index_template:
+        composed_of:
+        - metrics@tsdb-settings
+        - metrics-system.load@package
+        - metrics@custom
+        - system@custom
+        - metrics-system.load@custom
+        - ecs@mappings
+        - so-fleet_integrations.ip_mappings-1
+        - so-fleet_globals-1
+        - so-fleet_agent_id_verification-1
+        data_stream:
+          allow_custom_routing: false
+          hidden: false
+        ignore_missing_component_templates:
+        - metrics@custom
+        - system@custom
+        - metrics-system.load@custom
+        index_patterns:
+        - metrics-system.load-*
+        priority: 501
+        template:
+          settings:
+            index:
+              lifecycle:
+                name: so-metrics-system.load-logs
+              mode: time_series
+              number_of_replicas: 0
+      policy:
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 60d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
+    so-metrics-system_x_memory:
+      data_stream_lifecycle:
+        data_retention: 90d
+      index_template:
+        composed_of:
+        - metrics@tsdb-settings
+        - metrics-system.memory@package
+        - metrics@custom
+        - system@custom
+        - metrics-system.memory@custom
+        - ecs@mappings
+        - so-fleet_integrations.ip_mappings-1
+        - so-fleet_globals-1
+        - so-fleet_agent_id_verification-1
+        data_stream:
+          allow_custom_routing: false
+          hidden: false
+        ignore_missing_component_templates:
+        - metrics@custom
+        - system@custom
+        - metrics-system.memory@custom
+        index_patterns:
+        - metrics-system.memory-*
+        priority: 501
+        template:
+          settings:
+            index:
+              lifecycle:
+                name: so-metrics-system.memory-logs
+              mode: time_series
+              number_of_replicas: 0
+      policy:
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 60d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
+    so-metrics-system_x_network:
+      data_stream_lifecycle:
+        data_retention: 90d
+      index_template:
+        composed_of:
+        - metrics@tsdb-settings
+        - metrics-system.network@package
+        - metrics@custom
+        - system@custom
+        - metrics-system.network@custom
+        - ecs@mappings
+        - so-fleet_integrations.ip_mappings-1
+        - so-fleet_globals-1
+        - so-fleet_agent_id_verification-1
+        data_stream:
+          allow_custom_routing: false
+          hidden: false
+        ignore_missing_component_templates:
+        - metrics@custom
+        - system@custom
+        - metrics-system.network@custom
+        index_patterns:
+        - metrics-system.network-*
+        priority: 501
+        template:
+          settings:
+            index:
+              lifecycle:
+                name: so-metrics-system.network-logs
+              mode: time_series
+              number_of_replicas: 0
+      policy:
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 60d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
+    so-metrics-system_x_ntp:
+      data_stream_lifecycle:
+        data_retention: 90d
+      index_template:
+        composed_of:
+        - metrics@settings
+        - metrics-system.ntp@package
+        - metrics@custom
+        - system@custom
+        - metrics-system.ntp@custom
+        - ecs@mappings
+        - so-fleet_integrations.ip_mappings-1
+        - so-fleet_globals-1
+        - so-fleet_agent_id_verification-1
+        data_stream:
+          allow_custom_routing: false
+          hidden: false
+        ignore_missing_component_templates:
+        - metrics@custom
+        - system@custom
+        - metrics-system.ntp@custom
+        index_patterns:
+        - metrics-system.ntp-*
+        priority: 501
+        template:
+          settings:
+            index:
+              lifecycle:
+                name: so-metrics-system.ntp-logs
+              number_of_replicas: 0
+      policy:
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 60d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
+    so-metrics-system_x_process:
+      data_stream_lifecycle:
+        data_retention: 90d
+      index_template:
+        composed_of:
+        - metrics@tsdb-settings
+        - metrics-system.process@package
+        - metrics@custom
+        - system@custom
+        - metrics-system.process@custom
+        - ecs@mappings
+        - so-fleet_integrations.ip_mappings-1
+        - so-fleet_globals-1
+        - so-fleet_agent_id_verification-1
+        data_stream:
+          allow_custom_routing: false
+          hidden: false
+        ignore_missing_component_templates:
+        - metrics@custom
+        - system@custom
+        - metrics-system.process@custom
+        index_patterns:
+        - metrics-system.process-*
+        priority: 501
+        template:
+          settings:
+            index:
+              lifecycle:
+                name: so-metrics-system.process-logs
+              mode: time_series
+              number_of_replicas: 0
+      policy:
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 60d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
+    so-metrics-system_x_process_x_summary:
+      data_stream_lifecycle:
+        data_retention: 90d
+      index_template:
+        composed_of:
+        - metrics@tsdb-settings
+        - metrics-system.process.summary@package
+        - metrics@custom
+        - system@custom
+        - metrics-system.process.summary@custom
+        - ecs@mappings
+        - so-fleet_integrations.ip_mappings-1
+        - so-fleet_globals-1
+        - so-fleet_agent_id_verification-1
+        data_stream:
+          allow_custom_routing: false
+          hidden: false
+        ignore_missing_component_templates:
+        - metrics@custom
+        - system@custom
+        - metrics-system.process.summary@custom
+        index_patterns:
+        - metrics-system.process.summary-*
+        priority: 501
+        template:
+          settings:
+            index:
+              lifecycle:
+                name: so-metrics-system.process.summary-logs
+              mode: time_series
+              number_of_replicas: 0
+      policy:
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 60d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
+    so-metrics-system_x_socket_summary:
+      data_stream_lifecycle:
+        data_retention: 90d
+      index_template:
+        composed_of:
+        - metrics@tsdb-settings
+        - metrics-system.socket_summary@package
+        - metrics@custom
+        - system@custom
+        - metrics-system.socket_summary@custom
+        - ecs@mappings
+        - so-fleet_integrations.ip_mappings-1
+        - so-fleet_globals-1
+        - so-fleet_agent_id_verification-1
+        data_stream:
+          allow_custom_routing: false
+          hidden: false
+        ignore_missing_component_templates:
+        - metrics@custom
+        - system@custom
+        - metrics-system.socket_summary@custom
+        index_patterns:
+        - metrics-system.socket_summary-*
+        priority: 501
+        template:
+          settings:
+            index:
+              lifecycle:
+                name: so-metrics-system.socket_summary-logs
+              mode: time_series
+              number_of_replicas: 0
+      policy:
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 60d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
+    so-metrics-system_x_uptime:
+      data_stream_lifecycle:
+        data_retention: 90d
+      index_template:
+        composed_of:
+        - metrics@tsdb-settings
+        - metrics-system.uptime@package
+        - metrics@custom
+        - system@custom
+        - metrics-system.uptime@custom
+        - ecs@mappings
+        - so-fleet_integrations.ip_mappings-1
+        - so-fleet_globals-1
+        - so-fleet_agent_id_verification-1
+        data_stream:
+          allow_custom_routing: false
+          hidden: false
+        ignore_missing_component_templates:
+        - metrics@custom
+        - system@custom
+        - metrics-system.uptime@custom
+        index_patterns:
+        - metrics-system.uptime-*
+        priority: 501
+        template:
+          settings:
+            index:
+              lifecycle:
+                name: so-metrics-system.uptime-logs
+              mode: time_series
+              number_of_replicas: 0
+      policy:
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 60d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
     so-logs-windows_x_forwarded:
       index_sorting: false
       data_stream_lifecycle:

salt/elasticsearch/files/log4j2.properties

@@ -20,7 +20,8 @@ appender.rolling.strategy.type = DefaultRolloverStrategy
 appender.rolling.strategy.action.type = Delete
 appender.rolling.strategy.action.basepath = /var/log/elasticsearch
 appender.rolling.strategy.action.condition.type = IfFileName
-appender.rolling.strategy.action.condition.glob = *.log.gz
+# age delete regular securityonion.log.gz and gc.log.NN files
+appender.rolling.strategy.action.condition.regex = (?:.*[.]log[.]gz|gc[.]log[.][0-9]+)
 appender.rolling.strategy.action.condition.nested_condition.type = IfLastModified
 appender.rolling.strategy.action.condition.nested_condition.age = 7D
 

salt/elasticsearch/soc_elasticsearch.yaml

@@ -64,6 +64,43 @@ elasticsearch:
               flood_stage:
                 description: The max percentage of used disk space that will cause the node to take protective actions, such as blocking incoming events.
                 helpLink: elasticsearch
+      lifecycle:
+        default:
+          rollover:
+            description: This property accepts a key value pair formatted string and configures the conditions that would trigger a data stream to rollover when it has lifecycle configured.
+            forcedType: string
+            regex: ^max_age=(auto|[1-9][0-9]*[hd]),max_primary_shard_size=[1-9][0-9]*gb,min_docs=(0|[1-9][0-9]*),max_primary_shard_docs=[1-9][0-9]*$
+            regexFailureMessage: Must be in the format of "max_age=auto|<number><h|d>,max_primary_shard_size=<number>gb,min_docs=<number>,max_primary_shard_docs=<number>".
+            advanced: True
+            global: True
+    data_streams:
+      lifecycle:
+        poll_interval:
+          description: How often Elasticsearch checks what the next action is for all data streams with a built-in lifecycle.
+          forcedType: string
+          regex: "^[1-9][0-9]*[mhd]$"
+          regexFailureMessage: Must be a number followed by m, h, or d.
+          advanced: True
+          global: true
+          helpLink: elasticsearch
+        target:
+          merge:
+            policy:
+              merge_factor:
+                description: Data stream lifecycle implements tail merging by updating the Lucene merge policy factor for the target backing index. The merge factor is both the number of segments that should be merged together, and the maximum number of segments that we expect to find.
+                forcedType: int
+                regex: "^[1-9][0-9]*$"
+                advanced: True
+                global: true
+                helpLink: elasticsearch
+              floor_segment:
+                description: Data stream lifecycle implements tail merging by updating the Lucene merge policy floor segment for the target backing index. This floor segment size is a way to prevent indices from having a long tail of very small segments.
+                forcedType: string
+                regex: "^[1-9][0-9]*[MG]B$"
+                regexFailureMessage: Must be a number followed by MB or GB, such as 100MB.
+                advanced: True
+                global: true
+                helpLink: elasticsearch
     action:
       destructive_requires_name:
         description: Requires explicit index names when deleting indices. Prevents accidental deletion of indices via wildcard patterns.
@@ -645,6 +682,9 @@ elasticsearch:
             global: True
             advanced: True
             helpLink: elasticsearch
+    so-assistant-chat: *dataStreamSettings
+    so-assistant-session: *dataStreamSettings
+    so-elastic-agent-monitor: *dataStreamSettings
     so-logs-soc: *dataStreamSettings
     so-logs-system_x_auth: *dataStreamSettings
     so-logs-system_x_syslog: *dataStreamSettings
@@ -667,7 +707,10 @@ elasticsearch:
     so-logs-elastic_agent_x_auditbeat: *dataStreamSettings
     so-logs-elastic_agent_x_cloudbeat: *dataStreamSettings
     so-logs-elastic_agent_x_endpoint_security: *dataStreamSettings
+    so-logs-endpoint_x_actions: *dataStreamSettings
+    so-logs-endpoint_x_action_x_responses: *dataStreamSettings
     so-logs-endpoint_x_alerts: *dataStreamSettings
+    so-logs-endpoint_x_diagnostic_x_collection: *dataStreamSettings
     so-logs-endpoint_x_events_x_api: *dataStreamSettings
     so-logs-endpoint_x_events_x_file: *dataStreamSettings
     so-logs-endpoint_x_events_x_library: *dataStreamSettings
@@ -675,6 +718,7 @@ elasticsearch:
     so-logs-endpoint_x_events_x_process: *dataStreamSettings
     so-logs-endpoint_x_events_x_registry: *dataStreamSettings
     so-logs-endpoint_x_events_x_security: *dataStreamSettings
+    so-logs-endpoint_x_heartbeat: *dataStreamSettings
     so-logs-elastic_agent_x_filebeat: *dataStreamSettings
     so-logs-elastic_agent_x_fleet_server: *dataStreamSettings
     so-logs-elastic_agent_x_heartbeat: *dataStreamSettings
@@ -690,6 +734,19 @@ elasticsearch:
     so-metrics-vsphere_x_datastore: *dataStreamSettings
     so-metrics-vsphere_x_host: *dataStreamSettings
     so-metrics-vsphere_x_virtualmachine: *dataStreamSettings
+    so-metrics-system_x_core: *dataStreamSettings
+    so-metrics-system_x_cpu: *dataStreamSettings
+    so-metrics-system_x_diskio: *dataStreamSettings
+    so-metrics-system_x_filesystem: *dataStreamSettings
+    so-metrics-system_x_fsstat: *dataStreamSettings
+    so-metrics-system_x_load: *dataStreamSettings
+    so-metrics-system_x_memory: *dataStreamSettings
+    so-metrics-system_x_network: *dataStreamSettings
+    so-metrics-system_x_ntp: *dataStreamSettings
+    so-metrics-system_x_process: *dataStreamSettings
+    so-metrics-system_x_process_x_summary: *dataStreamSettings
+    so-metrics-system_x_socket_summary: *dataStreamSettings
+    so-metrics-system_x_uptime: *dataStreamSettings
     so-common: *dataStreamSettings
     so-endgame: *dataStreamSettings
     so-idh: *dataStreamSettings
@@ -880,17 +937,6 @@ elasticsearch:
                   global: True
                   advanced: True
                   helpLink: elasticsearch
-              rollover:
-                max_age:
-                  description: Maximum age of index.  Once an index reaches this limit, it will be rolled over into a new index.
-                  global: True
-                  advanced: True
-                  helpLink: elasticsearch
-                max_primary_shard_size:
-                  description: Maximum primary shard size. Once an index reaches this limit, it will be rolled over into a new index.
-                  global: True
-                  advanced: True
-                  helpLink: elasticsearch
               shrink:
                 method:
                   description: Shrink the index to a new index with fewer primary shards. Shrink operation is by count or size.
@@ -987,8 +1033,6 @@ elasticsearch:
             helpLink: elasticsearch
     sos-backup: *indexSettings
     so-detection: *indexSettings
-    so-assistant-chat: *indexSettings
-    so-assistant-session: *indexSettings
     so-metrics-fleet_server_x_agent_status: &fleetMetricsSettings
       index_sorting:
         description: Sorts the index by event time, at the cost of additional processing resource consumption.

salt/elasticsearch/template.map.jinja

@@ -109,9 +109,15 @@
 {%     if not settings.get('index_sorting', False) | to_bool and settings.index_template.template.settings.index.sort is defined %}
 {%       do settings.index_template.template.settings.index.pop('sort') %}
 {%     endif %}
-{%     if DATA_RETENTION_METHOD == 'DLM' and settings.index_template.data_stream is defined and settings.data_stream_lifecycle is defined %}
+{%     if DATA_RETENTION_METHOD == 'DLM' and settings.index_template.data_stream is defined %}
-{%       if settings.data_stream_lifecycle.data_retention is defined and settings.data_stream_lifecycle.data_retention %}
+{#       Addon defaults are generated without data_stream_lifecycle, so fall back to global defaults. #}
-{%         do settings.index_template.template.update({'lifecycle': {'data_retention': settings.data_stream_lifecycle.data_retention}}) %}
+{%       if settings.data_stream_lifecycle is defined %}
+{%         set DATA_STREAM_LIFECYCLE = settings.data_stream_lifecycle %}
+{%       else %}
+{%         set DATA_STREAM_LIFECYCLE = DEFAULT_GLOBAL_OVERRIDES.data_stream_lifecycle %}
+{%       endif %}
+{%       if DATA_STREAM_LIFECYCLE.data_retention is defined and DATA_STREAM_LIFECYCLE.data_retention %}
+{%         do settings.index_template.template.update({'lifecycle': {'data_retention': DATA_STREAM_LIFECYCLE.data_retention}}) %}
 {%       else %}
 {%         do settings.index_template.template.update({'lifecycle': {}}) %}
 {%       endif %}

salt/manager/tools/sbin/soup

@@ -442,7 +442,13 @@ get_soup_script_hashes() {
 }
 
 highstate() {
-  # Run a highstate.
+  # Run a highstate with a retry attempt.
+  if salt-call state.highstate -l info queue=True; then
+    return 0
+  fi
+
+  echo "Initial highstate attempt had a problem; retrying in 30 seconds."
+  sleep 30
   salt-call state.highstate -l info queue=True
 }
 
@@ -974,6 +980,10 @@ up_to_3.2.0() {
 
   pin_elasticsearch_data_retention_method
 
+  # Run so-elastic-fleet-es-url update with --force to ensure eval/import have
+  #  configured so-manager_elasicsearch as the default output for both monitoring and logs
+  /usr/sbin/so-elastic-fleet-es-url-update --force
+
   INSTALLEDVERSION=3.2.0
 }
 
@@ -1998,11 +2008,11 @@ main() {
     # Testing that salt-master is up by checking that is it connected to itself
     check_saltmaster_status
 
-    # update the salt-minion configs here and start the minion
+    # update the salt-master and salt-minion configs here and start the minion
     # since highstate are disabled above, minion start should not trigger a highstate
     echo ""
-    echo "Ensuring salt-minion configs are up-to-date."
+    echo "Ensuring salt-master and salt-minion configs are up-to-date."
-    salt-call state.apply salt.minion -l info queue=True
+    salt-call state.apply salt.master -l info queue=True
     echo ""
 
     # ensure the mine is updated and populated before highstates run, following the salt-master restart

salt/reactor/pillar_push_map.yaml

@@ -242,7 +242,7 @@ versionlock:
 # grain (compound supports nested grain matching via G@<key>:<subkey>:<value>).
 # pillar/vm/soc_vm.sls write path is referenced at salt/_runners/setup_hypervisor.py:856.
 vm:
-  - state: vm
+  - state: vm.user
     tgt: 'G@salt-cloud:driver:libvirt'
 
 # zeek: sensor_roles + so-import (5 roles).

salt/repo/client/oracle.sls

@@ -60,6 +60,9 @@ so_repo:
   {% endif %}
     - enabled: 1
     - gpgcheck: 1
+  {% if not GLOBALS.is_manager %}
+    - sslverify: 0
+  {% endif %}
 
 # Only assign the kernel repo once this node's running salt matches the version this
 # SO release ships. During a soup the grid is mid-salt-upgrade; gating here keeps the
@@ -77,6 +80,9 @@ so_kernel_repo:
   {% endif %}
     - enabled: 1
     - gpgcheck: 1
+  {% if not GLOBALS.is_manager %}
+    - sslverify: 0
+  {% endif %}
     # Supplementary kernel repo: tolerate it being empty/unreachable (e.g. before the
     # manager has populated /nsm/kernelrepo) so a missing repomd.xml can't make every
     # dnf/pkg operation on the grid fail.

salt/salt/defaults.yaml

@@ -6,4 +6,4 @@ salt:
     batch: '25%'
     batch_wait: 15
   schedule:
-    highstate_interval_hours: 2
+    highstate_interval_minutes: 120

salt/salt/highstate_schedule.sls

@@ -1,11 +1,15 @@
 {% from 'vars/globals.map.jinja' import GLOBALS %}
 {% from 'salt/schedule.map.jinja' import SCHEDULEMERGED %}
 
+{# splay a quarter of the interval, clamped to [5 min, 30 min], so short intervals
+   don't get jitter larger than the interval itself #}
+{% set SPLAY = [[(SCHEDULEMERGED.highstate_interval_minutes * 60 // 4) | int, 300] | max, 1800] | min %}
+
 highstate_schedule:
   schedule.present:
     - function: state.highstate
-    - hours: {{ SCHEDULEMERGED.highstate_interval_hours }}
+    - minutes: {{ SCHEDULEMERGED.highstate_interval_minutes }}
     - maxrunning: 1
 {% if not GLOBALS.is_manager %}
-    - splay: 1800
+    - splay: {{ SPLAY }}
 {% endif %}

salt/salt/master.sls

@@ -94,7 +94,9 @@ salt_master_service:
       - file: checkmine_engine
       - file: pillarWatch_engine
       - file: engines_config
-    - order: 9002
+    - require:
+      - cmd: wait_for_salt_minion_ready
+    - order: last
 
 {% else %}
 

salt/salt/soc_salt.yaml

@@ -31,9 +31,11 @@ salt:
       global: True
       advanced: True
   schedule:
-    highstate_interval_hours:
+    highstate_interval_minutes:
-      description: How often every minion in the grid runs a scheduled state.highstate, in hours. Lower values keep minions closer in sync at the cost of more load; higher values reduce load but increase worst-case latency for non-pushed changes. The salt-minion health check restarts a minion if its last highstate is older than this value plus one hour.
+      description: How often every minion in the grid runs a scheduled state.highstate, in minutes. Minimum 15 minutes. Lower values keep minions closer in sync at the cost of more load; higher values reduce load but increase worst-case latency for non-pushed changes. If Auto Apply is disabled, set this to the 15-minute minimum so changes are still picked up promptly. The salt-minion health check restarts a minion if its last state apply is older than this value plus one hour.
       forcedType: int
       helpLink: push
       global: True
       advanced: True
+      regex: '^(1[5-9]|[2-9][0-9]|[1-9][0-9]{2,4})$'
+      regexFailureMessage: The value must be an integer of at least 15 minutes (maximum 99999).

salt/soc/defaults.yaml

@@ -1512,6 +1512,10 @@ soc:
                 rulesetName: sos-resources-ag
                 branch: main
                 folder: securityonion-normalized
+              - repo: file:///nsm/airgap-resources/playbooks/securityonion-resources-playbooks
+                rulesetName: sos-published-ag
+                branch: published
+                folder: sigma
         assistant:
           systemPromptAddendum: ""
           systemPromptAddendumMaxLength: 50000
@@ -1530,9 +1534,9 @@ soc:
               healthTimeoutSeconds: 5
           agentic: false
           agentMapping:
-            Orchestrator: sonnet
+            Orchestrator: Claude Sonnet
-            Investigator: sonnet
+            Investigator: Claude Sonnet
-            Detection Engineer: sonnet
+            DetectionEngineer: Claude Sonnet
         onionconfig:
           saltstackDir: /opt/so/saltstack
           bypassEnabled: false
@@ -2714,4 +2718,14 @@ soc:
               enabled: true
               adapter: SOAI
               charsPerTokenEstimate: 4
+            - id: gemma
+              displayName: Gemma
+              origin: USA
+              contextLimitSmall: 256000
+              contextLimitLarge: 256000
+              lowBalanceColorAlert: 500000
+              enabled: true
+              adapter: SOAI
+              charsPerTokenEstimate: 4
+              
             

salt/soc/files/soc/playbook_placeholder_map.yaml

@@ -14,18 +14,22 @@
 
 CommandLine: process.command_line
 CurrentDirectory: process.working_directory
+Details: registry.data.strings
 Image: process.executable
-ImageLoaded: dll.name
+ImageLoaded: dll.path
 ParentImage: process.parent.executable
 ParentName: process.parent.name
 ParentProcessGuid: process.parent.entity_id
 ProcessGuid: process.entity_id
-TargetFilename: file.name
+TargetFilename: file.path
 TargetObject: registry.path
 TargetUserName: user.target.name
 User: user.name
 community_id: network.community_id
 dns_resolved_ip: dns.resolved_ip
+QueryName: dns.question.name
+dll_basename: dll.name
+file_basename: file.name
 document_id: soc_id
 dst_ip: destination.ip
 dst_port: destination.port
@@ -40,10 +44,15 @@ public_ip: network.public_ip
 related_hosts: related.hosts
 related_ip: related.ip
 src_ip: source.ip
-dns_query_name: dns.query_name
+dns_query_name: dns.question.name
 flow_id: log.id.uid
 payload: network.data.decoded
 rule_category: rule.category
 rule_name: rule.name
 rule_uuid: rule.uuid
 src_port: source.port
+# PowerShell channel (ps_script EID 4104 / classic EID 400)
+ScriptBlockText: powershell.file.script_block_text
+script_block_id: powershell.file.script_block_id
+script_block_hash: powershell.file.script_block_hash
+EngineVersion: powershell.engine.version

salt/soc/files/soc/sigma_playbook_pipeline.yaml

@@ -1,8 +1,9 @@
 name: Security Onion - Playbook Pipeline
 priority: 97
 transformations:
-    # Route string fields to their lowercase-normalized .caseless subfield so wildcard
+    # Route to lowercase-normalized .caseless subfields for case-insensitive matching.
-    # matches are case-insensitive.
+    # file.path.caseless exists on Defend only (Sysmon file events lack it);
+    # registry.path / dll.path / file.name have no .caseless on any source.
     - id: case_insensitive_string_fields
       type: field_name_mapping
       mapping:
@@ -10,3 +11,145 @@ transformations:
         process.parent.executable: process.parent.executable.caseless
         process.command_line: process.command_line.caseless
         process.parent.command_line: process.parent.command_line.caseless
+        file.path: file.path.caseless
+    # entity_id pivots must also match processes that were already running when the
+    # agent started: Defend emits already_running (event.type:info), not start.
+    # Kept out of Playbook sigma query because Sysmon has no equivalent concept.
+    # contains_field is exact, so child pivots (process.parent.entity_id) stay
+    # start-only. Drop must precede add;
+    - id: playbook_process_lifecycle_drop_start_scope
+      type: drop_detection_item
+      field_name_conditions:
+        - type: include_fields
+          fields: ['event.type']
+      rule_conditions:
+        - type: logsource
+          category: process_creation
+        - type: contains_field
+          field: process.entity_id
+    - id: playbook_process_lifecycle_add-fields
+      type: add_condition
+      conditions:
+        event.type: ['start', 'info']
+      rule_conditions:
+        - type: logsource
+          category: process_creation
+        - type: contains_field
+          field: process.entity_id
+    # file_activity: playbook-only pseudo-category spanning all file operations.
+    - id: playbook_file_activity_add-fields
+      type: add_condition
+      conditions:
+        event.category: 'file'
+      rule_conditions:
+      - type: logsource
+        category: file_activity
+    # EventType -> event.action is safe as a projection only — the values differ
+    # (Sysmon DeleteFile/SetValue vs ECS deletion/modification).
+    - id: playbook_file_activity_event_type
+      type: field_name_mapping
+      mapping:
+        winlog.event_data.EventType: event.action
+      rule_conditions:
+      - type: logsource
+        category: file_activity
+    - id: playbook_file_event_event_type
+      type: field_name_mapping
+      mapping:
+        winlog.event_data.EventType: event.action
+      rule_conditions:
+      - type: logsource
+        category: file_event
+    - id: playbook_file_delete_event_type
+      type: field_name_mapping
+      mapping:
+        winlog.event_data.EventType: event.action
+      rule_conditions:
+      - type: logsource
+        category: file_delete
+    - id: playbook_file_rename_event_type
+      type: field_name_mapping
+      mapping:
+        winlog.event_data.EventType: event.action
+      rule_conditions:
+      - type: logsource
+        category: file_rename
+    - id: playbook_registry_set_event_type
+      type: field_name_mapping
+      mapping:
+        winlog.event_data.EventType: event.action
+      rule_conditions:
+      - type: logsource
+        category: registry_set
+    # WMI-Activity events carry no event.category; event.dataset is source-specific
+    - id: playbook_wmi_activity_add-fields
+      type: add_condition
+      conditions:
+        event.dataset: 'wmi.operational'
+      rule_conditions:
+      - type: logsource
+        category: wmi
+    - id: playbook_wmi_event_activity_add-fields
+      type: add_condition
+      conditions:
+        event.dataset: 'wmi.operational'
+      rule_conditions:
+      - type: logsource
+        category: wmi_event
+    - id: playbook_ps_script_field-mapping
+      type: field_name_mapping
+      mapping:
+        ScriptBlockText: powershell.file.script_block_text
+        Path: file.path
+      rule_conditions:
+      - type: logsource
+        category: ps_script
+    - id: playbook_ps_script_add-fields
+      type: add_condition
+      conditions:
+        event.dataset: 'windows.powershell_operational'
+        event.code: '4104'
+      rule_conditions:
+      - type: logsource
+        category: ps_script
+    # Data (raw EID 400/600 message blob) -> process.command_line: the winlog
+    # integration parses HostApplication into it; `message` is analyzed text and
+    # wildcard matches on it silently return zero. No .caseless on this dataset.
+    - id: playbook_ps_classic_start_field-mapping
+      type: field_name_mapping
+      mapping:
+        Data: process.command_line
+      rule_conditions:
+      - type: logsource
+        category: ps_classic_start
+    - id: playbook_ps_classic_start_add-fields
+      type: add_condition
+      conditions:
+        event.dataset: 'windows.powershell'
+        event.code: '400'
+      rule_conditions:
+      - type: logsource
+        category: ps_classic_start
+    - id: playbook_ps_classic_provider_start_field-mapping
+      type: field_name_mapping
+      mapping:
+        Data: process.command_line
+      rule_conditions:
+      - type: logsource
+        category: ps_classic_provider_start
+    - id: playbook_ps_classic_provider_start_add-fields
+      type: add_condition
+      conditions:
+        event.dataset: 'windows.powershell'
+        event.code: '600'
+      rule_conditions:
+      - type: logsource
+        category: ps_classic_provider_start
+    # Alert docs nest the host under event_data.host.name; no top-level host.name.
+    - id: playbook_alert_host_field
+      type: field_name_mapping
+      mapping:
+        host.name: event_data.host.name
+      rule_conditions:
+      - type: logsource
+        category: alert

salt/soc/files/soc/sigma_so_pipeline.yaml

@@ -46,31 +46,47 @@ transformations:
       - type: logsource
         product: opencanary
     # Maps "antivirus" category to Windows Defender logs shipped by Elastic Agent Winlog Integration
+    # and to Elastic Defend malware alerts
     # winlog.event_data.threat_name has to be renamed prior to ingestion, it is originally winlog.event_data.Threat Name
-    - id: antivirus_field-mappings_windows-defender
+    - id: antivirus_field-mappings
       type: field_name_mapping
       mapping:
-        Signature: winlog.event_data.threat_name
+        Signature:
+          - winlog.event_data.threat_name
+          - rule.name
       rule_conditions:
       - type: logsource
         category: antivirus
-    - id: antivirus_add-fields_windows-defender
+    - id: antivirus_add-fields
       type: add_condition
       conditions:
-        winlog.channel: 'Microsoft-Windows-Windows Defender/Operational'
+        event.code:
-        winlog.provider_name: 'Microsoft-Windows-Windows Defender'
+          - "1116"
-        event.code: "1116"
+          - "malicious_file"
+          - "memory_signature"
       rule_conditions:
       - type: logsource
         category: antivirus
     # OS-agnostic process_creation scoping for product-less (NIDS/host-pivot) rules.
+    # pySigma: rule_cond_expr requires rule_conditions as a mapping, not a list.
     - id: process_creation_os_agnostic
       type: add_condition
       conditions:
         event.category: process
       rule_conditions:
-      - type: logsource
+        pc_cat:
-        category: process_creation
+          type: logsource
+          category: process_creation
+        pc_win:
+          type: logsource
+          product: windows
+        pc_mac:
+          type: logsource
+          product: macos
+        pc_lin:
+          type: logsource
+          product: linux
+      rule_cond_expr: "pc_cat and not (pc_win or pc_mac or pc_lin)"
     # Transforms the `Hashes` field to ECS fields
     # ECS fields are used by the hash fields emitted by Elastic Defend
     # If shipped with Elastic Agent, sysmon logs will also have hashes mapped to ECS fields    
@@ -131,13 +147,22 @@ transformations:
       type: field_name_mapping
       mapping:
         winlog.event_data.Details: registry.data.strings
-        # field rename only; EventType values (SetValue/CreateKey) still differ from
-        # event.action values (modification/creation)
-        winlog.event_data.EventType: event.action
       rule_conditions:
       - type: logsource
         product: windows
         category: registry_set
+    # ecs_windows renames Initiated -> network.direction without translating the value,
+    # so rules compile to network.direction:"true" and never match (field holds
+    # ingress/egress). network.initiated carries the boolean on both Defend and Sysmon.
+    # Keyed on network.direction because ecs_windows has already renamed by this layer.
+    - id: ecs_fix_network_connection_initiated
+      type: field_name_mapping
+      mapping:
+        network.direction: network.initiated
+      rule_conditions:
+      - type: logsource
+        product: windows
+        category: network_connection
     - id: ecs_fix_image_load
       type: field_name_mapping
       mapping:
@@ -150,6 +175,30 @@ transformations:
       - type: logsource
         product: windows
         category: image_load
+    # Defend reports the driver image in dll.path (file.path is null on driver events) —
+    # same renames as image_load.
+    - id: ecs_fix_driver_load
+      type: field_name_mapping
+      mapping:
+        file.path: dll.path
+        file.code_signature.signed: dll.code_signature.exists
+        winlog.event_data.Signature: dll.code_signature.subject_name
+        file.code_signature.status: dll.code_signature.status
+        winlog.event_data.Hashes: dll.hash.sha256
+      rule_conditions:
+      - type: logsource
+        product: windows
+        category: driver_load
+    - id: ecs_fix_file_rename
+      type: field_name_mapping
+      mapping:
+        # Defend records the pre-rename path in file.Ext.original.path; Sysmon's
+        # SourceFilename has no ECS equivalent.
+        winlog.event_data.SourceFilename: file.Ext.original.path
+      rule_conditions:
+      - type: logsource
+        product: windows
+        category: file_rename
     - id: linux_security_add-fields
       type: add_condition
       conditions:
@@ -323,6 +372,33 @@ transformations:
       rule_conditions:
       - type: logsource
         category: file_event
+    # Without event.type scoping, file_delete rules also match creations.
+    - id: endpoint_file_delete_add-fields
+      type: add_condition
+      conditions:
+        event.category: 'file'
+        event.type: 'deletion'
+      rule_conditions:
+      - type: logsource
+        category: file_delete
+    # Defend reports renames as event.action:rename with event.type:change.
+    - id: endpoint_file_rename_add-fields
+      type: add_condition
+      conditions:
+        event.category: 'file'
+        event.action: 'rename'
+      rule_conditions:
+      - type: logsource
+        category: file_rename
+    # event.type:change selects writes and excludes Defend's read-only registry events.
+    - id: endpoint_registry_set_add-fields
+      type: add_condition
+      conditions:
+        event.category: 'registry'
+        event.type: 'change'
+      rule_conditions:
+      - type: logsource
+        category: registry_set
     # Scope image_load rules to Elastic Endpoint library events (event.category:library, dll.*
     # populated).
     - id: endpoint_image_load_add-fields
@@ -351,6 +427,149 @@ transformations:
       rule_conditions:
       - type: logsource
         category: network_connection
+    # Scope on lookup actions, not network.protocol:dns (also matches Zeek port-53).
+    - id: endpoint_dns_query_add-fields
+      type: add_condition
+      conditions:
+        event.category: 'network'
+        event.action:
+          - 'lookup_requested'
+          - 'lookup_result'
+      rule_conditions:
+      - type: logsource
+        category: dns_query
+    # OS gates: without them a `product: windows` rule in these categories also matches
+    # Linux/macOS. Separate conditions (not `product:` on the mappings above) so the
+    # mappings keep scoping product-less rules.
+    - id: endpoint_file_create_windows_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'windows'
+      rule_conditions:
+      - type: logsource
+        category: file_event
+        product: windows
+    - id: endpoint_file_create_linux_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'linux'
+      rule_conditions:
+      - type: logsource
+        category: file_event
+        product: linux
+    - id: endpoint_file_create_macos_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'macos'
+      rule_conditions:
+      - type: logsource
+        category: file_event
+        product: macos
+    - id: endpoint_file_delete_windows_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'windows'
+      rule_conditions:
+      - type: logsource
+        category: file_delete
+        product: windows
+    - id: endpoint_file_delete_linux_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'linux'
+      rule_conditions:
+      - type: logsource
+        category: file_delete
+        product: linux
+    - id: endpoint_file_delete_macos_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'macos'
+      rule_conditions:
+      - type: logsource
+        category: file_delete
+        product: macos
+    - id: endpoint_file_rename_windows_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'windows'
+      rule_conditions:
+      - type: logsource
+        category: file_rename
+        product: windows
+    - id: endpoint_file_rename_linux_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'linux'
+      rule_conditions:
+      - type: logsource
+        category: file_rename
+        product: linux
+    - id: endpoint_file_rename_macos_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'macos'
+      rule_conditions:
+      - type: logsource
+        category: file_rename
+        product: macos
+    - id: endpoint_network_connection_windows_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'windows'
+      rule_conditions:
+      - type: logsource
+        category: network_connection
+        product: windows
+    - id: endpoint_network_connection_linux_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'linux'
+      rule_conditions:
+      - type: logsource
+        category: network_connection
+        product: linux
+    - id: endpoint_network_connection_macos_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'macos'
+      rule_conditions:
+      - type: logsource
+        category: network_connection
+        product: macos
+    - id: endpoint_dns_query_windows_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'windows'
+      rule_conditions:
+      - type: logsource
+        category: dns_query
+        product: windows
+    - id: endpoint_dns_query_linux_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'linux'
+      rule_conditions:
+      - type: logsource
+        category: dns_query
+        product: linux
+    - id: endpoint_dns_query_macos_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'macos'
+      rule_conditions:
+      - type: logsource
+        category: dns_query
+        product: macos
+    # registry_set / image_load / driver_load are not gated — windows-only telemetry.
+    # wmi / wmi_event are not scoped here: their only handle is event.dataset
+    - id: endpoint_driver_load_add-fields
+      type: add_condition
+      conditions:
+        event.category: 'driver'
+      rule_conditions:
+      - type: logsource
+        category: driver_load
     # Maps "alert" category to SO Alert events
     - id: alert_so_add-fields
       type: add_condition
@@ -423,4 +642,4 @@ transformations:
         tags: '*file'
       rule_conditions:
       - type: logsource
-        category: file
+        category: file

salt/soc/soc_soc.yaml

@@ -786,7 +786,7 @@ soc:
             Investigator:
               description: This agent investigates alerts, explains events and records, and hunts through event data. It can also acknowledge alerts and escalate to cases.
               global: True
-            Detection Engineer:
+            DetectionEngineer:
               description: This agent manages detections and their overrides, including tuning noisy rules and authoring rule content.
               global: True
       client:

salt/vars/elasticsearch.map.jinja

@@ -1,10 +1,11 @@
 {% import 'vars/init.map.jinja' as INIT %}
+{% import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS %}
 
 {%
   set ELASTICSEARCH_GLOBALS = {
     'elasticsearch': {
       'es_cluster_name': INIT.PILLAR.elasticsearch.config.cluster.name,
-      'es_heap': INIT.PILLAR.elasticsearch.esheap
+      'es_heap': INIT.PILLAR.elasticsearch.get('esheap', ELASTICSEARCHDEFAULTS.elasticsearch.esheap)
     }
   }
 %}

salt/zeek/defaults.yaml

@@ -62,7 +62,6 @@ zeek:
         - securityonion/file-extraction
         - securityonion/community-id-extended
         - oui-logging
-        - icsnpp-modbus
         - icsnpp-dnp3
         - icsnpp-bacnet
         - icsnpp-ethercat