Remove icsnpp-modbus from defaults.yaml

Removed 'icsnpp-modbus' from the list of modules.

salt/common/tools/sbin/so-log-check

@@ -133,6 +133,7 @@ if [[ $EXCLUDE_STARTUP_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Cancelling deferred write event maybeFenceReplicas because the event queue is now closed" # Kafka controller log during shutdown/restart
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Redis may have been restarted" # Redis likely restarted by salt
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|file already closed"          # Go logging race condition during container restart
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|relation \"audit_settings\" does not exist" # salt checking for changes before SOC starts
 fi
 
 if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade

@@ -17,7 +17,7 @@ if INSTALLED_VERSION=$(elastic_fleet_package_version_check "{{ PACKAGE }}") && L
     if [ "$INSTALLED_VERSION" == "$LATEST_VERSION" ]; then
         echo "{{ PACKAGE }} integration version $INSTALLED_VERSION is already at the reported latest version $LATEST_VERSION, skipping upgrade."
     else
-        echo "Upgrading {{ PACKAGE }} package to version $LATEST_VERSION..."
+        echo "Upgrading {{ PACKAGE }} package from $INSTALLED_VERSION to version $LATEST_VERSION..."
         if ! elastic_fleet_package_install "{{ PACKAGE }}" "$LATEST_VERSION"; then
             PKG_LOAD_FAILURES=$((PKG_LOAD_FAILURES + 1))
             PKG_LOAD_FAILURES_NAMES+=("{{ PACKAGE }}")

salt/elasticsearch/defaults.yaml

@@ -3453,6 +3453,720 @@ elasticsearch:
               set_priority:
                 priority: 50
             min_age: 30d
+    so-metrics-system_x_core:
+      data_stream_lifecycle:
+        data_retention: 90d
+      index_template:
+        composed_of:
+        - metrics@tsdb-settings
+        - metrics-system.core@package
+        - metrics@custom
+        - system@custom
+        - metrics-system.core@custom
+        - ecs@mappings
+        - so-fleet_integrations.ip_mappings-1
+        - so-fleet_globals-1
+        - so-fleet_agent_id_verification-1
+        data_stream:
+          allow_custom_routing: false
+          hidden: false
+        ignore_missing_component_templates:
+        - metrics@custom
+        - system@custom
+        - metrics-system.core@custom
+        index_patterns:
+        - metrics-system.core-*
+        priority: 501
+        template:
+          settings:
+            index:
+              lifecycle:
+                name: so-metrics-system.core-logs
+              mode: time_series
+              number_of_replicas: 0
+      policy:
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 60d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
+    so-metrics-system_x_cpu:
+      data_stream_lifecycle:
+        data_retention: 90d
+      index_template:
+        composed_of:
+        - metrics@tsdb-settings
+        - metrics-system.cpu@package
+        - metrics@custom
+        - system@custom
+        - metrics-system.cpu@custom
+        - ecs@mappings
+        - so-fleet_integrations.ip_mappings-1
+        - so-fleet_globals-1
+        - so-fleet_agent_id_verification-1
+        data_stream:
+          allow_custom_routing: false
+          hidden: false
+        ignore_missing_component_templates:
+        - metrics@custom
+        - system@custom
+        - metrics-system.cpu@custom
+        index_patterns:
+        - metrics-system.cpu-*
+        priority: 501
+        template:
+          settings:
+            index:
+              lifecycle:
+                name: so-metrics-system.cpu-logs
+              mode: time_series
+              number_of_replicas: 0
+      policy:
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 60d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
+    so-metrics-system_x_diskio:
+      data_stream_lifecycle:
+        data_retention: 90d
+      index_template:
+        composed_of:
+        - metrics@tsdb-settings
+        - metrics-system.diskio@package
+        - metrics@custom
+        - system@custom
+        - metrics-system.diskio@custom
+        - ecs@mappings
+        - so-fleet_integrations.ip_mappings-1
+        - so-fleet_globals-1
+        - so-fleet_agent_id_verification-1
+        data_stream:
+          allow_custom_routing: false
+          hidden: false
+        ignore_missing_component_templates:
+        - metrics@custom
+        - system@custom
+        - metrics-system.diskio@custom
+        index_patterns:
+        - metrics-system.diskio-*
+        priority: 501
+        template:
+          settings:
+            index:
+              lifecycle:
+                name: so-metrics-system.diskio-logs
+              mode: time_series
+              number_of_replicas: 0
+      policy:
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 60d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
+    so-metrics-system_x_filesystem:
+      data_stream_lifecycle:
+        data_retention: 90d
+      index_template:
+        composed_of:
+        - metrics@tsdb-settings
+        - metrics-system.filesystem@package
+        - metrics@custom
+        - system@custom
+        - metrics-system.filesystem@custom
+        - ecs@mappings
+        - so-fleet_integrations.ip_mappings-1
+        - so-fleet_globals-1
+        - so-fleet_agent_id_verification-1
+        data_stream:
+          allow_custom_routing: false
+          hidden: false
+        ignore_missing_component_templates:
+        - metrics@custom
+        - system@custom
+        - metrics-system.filesystem@custom
+        index_patterns:
+        - metrics-system.filesystem-*
+        priority: 501
+        template:
+          settings:
+            index:
+              lifecycle:
+                name: so-metrics-system.filesystem-logs
+              mode: time_series
+              number_of_replicas: 0
+      policy:
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 60d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
+    so-metrics-system_x_fsstat:
+      data_stream_lifecycle:
+        data_retention: 90d
+      index_template:
+        composed_of:
+        - metrics@tsdb-settings
+        - metrics-system.fsstat@package
+        - metrics@custom
+        - system@custom
+        - metrics-system.fsstat@custom
+        - ecs@mappings
+        - so-fleet_integrations.ip_mappings-1
+        - so-fleet_globals-1
+        - so-fleet_agent_id_verification-1
+        data_stream:
+          allow_custom_routing: false
+          hidden: false
+        ignore_missing_component_templates:
+        - metrics@custom
+        - system@custom
+        - metrics-system.fsstat@custom
+        index_patterns:
+        - metrics-system.fsstat-*
+        priority: 501
+        template:
+          settings:
+            index:
+              lifecycle:
+                name: so-metrics-system.fsstat-logs
+              mode: time_series
+              number_of_replicas: 0
+      policy:
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 60d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
+    so-metrics-system_x_load:
+      data_stream_lifecycle:
+        data_retention: 90d
+      index_template:
+        composed_of:
+        - metrics@tsdb-settings
+        - metrics-system.load@package
+        - metrics@custom
+        - system@custom
+        - metrics-system.load@custom
+        - ecs@mappings
+        - so-fleet_integrations.ip_mappings-1
+        - so-fleet_globals-1
+        - so-fleet_agent_id_verification-1
+        data_stream:
+          allow_custom_routing: false
+          hidden: false
+        ignore_missing_component_templates:
+        - metrics@custom
+        - system@custom
+        - metrics-system.load@custom
+        index_patterns:
+        - metrics-system.load-*
+        priority: 501
+        template:
+          settings:
+            index:
+              lifecycle:
+                name: so-metrics-system.load-logs
+              mode: time_series
+              number_of_replicas: 0
+      policy:
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 60d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
+    so-metrics-system_x_memory:
+      data_stream_lifecycle:
+        data_retention: 90d
+      index_template:
+        composed_of:
+        - metrics@tsdb-settings
+        - metrics-system.memory@package
+        - metrics@custom
+        - system@custom
+        - metrics-system.memory@custom
+        - ecs@mappings
+        - so-fleet_integrations.ip_mappings-1
+        - so-fleet_globals-1
+        - so-fleet_agent_id_verification-1
+        data_stream:
+          allow_custom_routing: false
+          hidden: false
+        ignore_missing_component_templates:
+        - metrics@custom
+        - system@custom
+        - metrics-system.memory@custom
+        index_patterns:
+        - metrics-system.memory-*
+        priority: 501
+        template:
+          settings:
+            index:
+              lifecycle:
+                name: so-metrics-system.memory-logs
+              mode: time_series
+              number_of_replicas: 0
+      policy:
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 60d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
+    so-metrics-system_x_network:
+      data_stream_lifecycle:
+        data_retention: 90d
+      index_template:
+        composed_of:
+        - metrics@tsdb-settings
+        - metrics-system.network@package
+        - metrics@custom
+        - system@custom
+        - metrics-system.network@custom
+        - ecs@mappings
+        - so-fleet_integrations.ip_mappings-1
+        - so-fleet_globals-1
+        - so-fleet_agent_id_verification-1
+        data_stream:
+          allow_custom_routing: false
+          hidden: false
+        ignore_missing_component_templates:
+        - metrics@custom
+        - system@custom
+        - metrics-system.network@custom
+        index_patterns:
+        - metrics-system.network-*
+        priority: 501
+        template:
+          settings:
+            index:
+              lifecycle:
+                name: so-metrics-system.network-logs
+              mode: time_series
+              number_of_replicas: 0
+      policy:
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 60d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
+    so-metrics-system_x_ntp:
+      data_stream_lifecycle:
+        data_retention: 90d
+      index_template:
+        composed_of:
+        - metrics@settings
+        - metrics-system.ntp@package
+        - metrics@custom
+        - system@custom
+        - metrics-system.ntp@custom
+        - ecs@mappings
+        - so-fleet_integrations.ip_mappings-1
+        - so-fleet_globals-1
+        - so-fleet_agent_id_verification-1
+        data_stream:
+          allow_custom_routing: false
+          hidden: false
+        ignore_missing_component_templates:
+        - metrics@custom
+        - system@custom
+        - metrics-system.ntp@custom
+        index_patterns:
+        - metrics-system.ntp-*
+        priority: 501
+        template:
+          settings:
+            index:
+              lifecycle:
+                name: so-metrics-system.ntp-logs
+              number_of_replicas: 0
+      policy:
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 60d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
+    so-metrics-system_x_process:
+      data_stream_lifecycle:
+        data_retention: 90d
+      index_template:
+        composed_of:
+        - metrics@tsdb-settings
+        - metrics-system.process@package
+        - metrics@custom
+        - system@custom
+        - metrics-system.process@custom
+        - ecs@mappings
+        - so-fleet_integrations.ip_mappings-1
+        - so-fleet_globals-1
+        - so-fleet_agent_id_verification-1
+        data_stream:
+          allow_custom_routing: false
+          hidden: false
+        ignore_missing_component_templates:
+        - metrics@custom
+        - system@custom
+        - metrics-system.process@custom
+        index_patterns:
+        - metrics-system.process-*
+        priority: 501
+        template:
+          settings:
+            index:
+              lifecycle:
+                name: so-metrics-system.process-logs
+              mode: time_series
+              number_of_replicas: 0
+      policy:
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 60d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
+    so-metrics-system_x_process_x_summary:
+      data_stream_lifecycle:
+        data_retention: 90d
+      index_template:
+        composed_of:
+        - metrics@tsdb-settings
+        - metrics-system.process.summary@package
+        - metrics@custom
+        - system@custom
+        - metrics-system.process.summary@custom
+        - ecs@mappings
+        - so-fleet_integrations.ip_mappings-1
+        - so-fleet_globals-1
+        - so-fleet_agent_id_verification-1
+        data_stream:
+          allow_custom_routing: false
+          hidden: false
+        ignore_missing_component_templates:
+        - metrics@custom
+        - system@custom
+        - metrics-system.process.summary@custom
+        index_patterns:
+        - metrics-system.process.summary-*
+        priority: 501
+        template:
+          settings:
+            index:
+              lifecycle:
+                name: so-metrics-system.process.summary-logs
+              mode: time_series
+              number_of_replicas: 0
+      policy:
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 60d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
+    so-metrics-system_x_socket_summary:
+      data_stream_lifecycle:
+        data_retention: 90d
+      index_template:
+        composed_of:
+        - metrics@tsdb-settings
+        - metrics-system.socket_summary@package
+        - metrics@custom
+        - system@custom
+        - metrics-system.socket_summary@custom
+        - ecs@mappings
+        - so-fleet_integrations.ip_mappings-1
+        - so-fleet_globals-1
+        - so-fleet_agent_id_verification-1
+        data_stream:
+          allow_custom_routing: false
+          hidden: false
+        ignore_missing_component_templates:
+        - metrics@custom
+        - system@custom
+        - metrics-system.socket_summary@custom
+        index_patterns:
+        - metrics-system.socket_summary-*
+        priority: 501
+        template:
+          settings:
+            index:
+              lifecycle:
+                name: so-metrics-system.socket_summary-logs
+              mode: time_series
+              number_of_replicas: 0
+      policy:
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 60d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
+    so-metrics-system_x_uptime:
+      data_stream_lifecycle:
+        data_retention: 90d
+      index_template:
+        composed_of:
+        - metrics@tsdb-settings
+        - metrics-system.uptime@package
+        - metrics@custom
+        - system@custom
+        - metrics-system.uptime@custom
+        - ecs@mappings
+        - so-fleet_integrations.ip_mappings-1
+        - so-fleet_globals-1
+        - so-fleet_agent_id_verification-1
+        data_stream:
+          allow_custom_routing: false
+          hidden: false
+        ignore_missing_component_templates:
+        - metrics@custom
+        - system@custom
+        - metrics-system.uptime@custom
+        index_patterns:
+        - metrics-system.uptime-*
+        priority: 501
+        template:
+          settings:
+            index:
+              lifecycle:
+                name: so-metrics-system.uptime-logs
+              mode: time_series
+              number_of_replicas: 0
+      policy:
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 60d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
     so-logs-windows_x_forwarded:
       index_sorting: false
       data_stream_lifecycle:

salt/elasticsearch/files/log4j2.properties

@@ -20,7 +20,8 @@ appender.rolling.strategy.type = DefaultRolloverStrategy
 appender.rolling.strategy.action.type = Delete
 appender.rolling.strategy.action.basepath = /var/log/elasticsearch
 appender.rolling.strategy.action.condition.type = IfFileName
-appender.rolling.strategy.action.condition.glob = *.log.gz
+# age delete regular securityonion.log.gz and gc.log.NN files
+appender.rolling.strategy.action.condition.regex = (?:.*[.]log[.]gz|gc[.]log[.][0-9]+)
 appender.rolling.strategy.action.condition.nested_condition.type = IfLastModified
 appender.rolling.strategy.action.condition.nested_condition.age = 7D
 

salt/elasticsearch/soc_elasticsearch.yaml

@@ -64,6 +64,43 @@ elasticsearch:
               flood_stage:
                 description: The max percentage of used disk space that will cause the node to take protective actions, such as blocking incoming events.
                 helpLink: elasticsearch
+      lifecycle:
+        default:
+          rollover:
+            description: This property accepts a key value pair formatted string and configures the conditions that would trigger a data stream to rollover when it has lifecycle configured.
+            forcedType: string
+            regex: ^max_age=(auto|[1-9][0-9]*[hd]),max_primary_shard_size=[1-9][0-9]*gb,min_docs=(0|[1-9][0-9]*),max_primary_shard_docs=[1-9][0-9]*$
+            regexFailureMessage: Must be in the format of "max_age=auto|<number><h|d>,max_primary_shard_size=<number>gb,min_docs=<number>,max_primary_shard_docs=<number>".
+            advanced: True
+            global: True
+    data_streams:
+      lifecycle:
+        poll_interval:
+          description: How often Elasticsearch checks what the next action is for all data streams with a built-in lifecycle.
+          forcedType: string
+          regex: "^[1-9][0-9]*[mhd]$"
+          regexFailureMessage: Must be a number followed by m, h, or d.
+          advanced: True
+          global: true
+          helpLink: elasticsearch
+        target:
+          merge:
+            policy:
+              merge_factor:
+                description: Data stream lifecycle implements tail merging by updating the Lucene merge policy factor for the target backing index. The merge factor is both the number of segments that should be merged together, and the maximum number of segments that we expect to find.
+                forcedType: int
+                regex: "^[1-9][0-9]*$"
+                advanced: True
+                global: true
+                helpLink: elasticsearch
+              floor_segment:
+                description: Data stream lifecycle implements tail merging by updating the Lucene merge policy floor segment for the target backing index. This floor segment size is a way to prevent indices from having a long tail of very small segments.
+                forcedType: string
+                regex: "^[1-9][0-9]*[MG]B$"
+                regexFailureMessage: Must be a number followed by MB or GB, such as 100MB.
+                advanced: True
+                global: true
+                helpLink: elasticsearch
     action:
       destructive_requires_name:
         description: Requires explicit index names when deleting indices. Prevents accidental deletion of indices via wildcard patterns.
@@ -645,6 +682,9 @@ elasticsearch:
             global: True
             advanced: True
             helpLink: elasticsearch
+    so-assistant-chat: *dataStreamSettings
+    so-assistant-session: *dataStreamSettings
+    so-elastic-agent-monitor: *dataStreamSettings
     so-logs-soc: *dataStreamSettings
     so-logs-system_x_auth: *dataStreamSettings
     so-logs-system_x_syslog: *dataStreamSettings
@@ -667,7 +707,10 @@ elasticsearch:
     so-logs-elastic_agent_x_auditbeat: *dataStreamSettings
     so-logs-elastic_agent_x_cloudbeat: *dataStreamSettings
     so-logs-elastic_agent_x_endpoint_security: *dataStreamSettings
+    so-logs-endpoint_x_actions: *dataStreamSettings
+    so-logs-endpoint_x_action_x_responses: *dataStreamSettings
     so-logs-endpoint_x_alerts: *dataStreamSettings
+    so-logs-endpoint_x_diagnostic_x_collection: *dataStreamSettings
     so-logs-endpoint_x_events_x_api: *dataStreamSettings
     so-logs-endpoint_x_events_x_file: *dataStreamSettings
     so-logs-endpoint_x_events_x_library: *dataStreamSettings
@@ -675,6 +718,7 @@ elasticsearch:
     so-logs-endpoint_x_events_x_process: *dataStreamSettings
     so-logs-endpoint_x_events_x_registry: *dataStreamSettings
     so-logs-endpoint_x_events_x_security: *dataStreamSettings
+    so-logs-endpoint_x_heartbeat: *dataStreamSettings
     so-logs-elastic_agent_x_filebeat: *dataStreamSettings
     so-logs-elastic_agent_x_fleet_server: *dataStreamSettings
     so-logs-elastic_agent_x_heartbeat: *dataStreamSettings
@@ -690,6 +734,19 @@ elasticsearch:
     so-metrics-vsphere_x_datastore: *dataStreamSettings
     so-metrics-vsphere_x_host: *dataStreamSettings
     so-metrics-vsphere_x_virtualmachine: *dataStreamSettings
+    so-metrics-system_x_core: *dataStreamSettings
+    so-metrics-system_x_cpu: *dataStreamSettings
+    so-metrics-system_x_diskio: *dataStreamSettings
+    so-metrics-system_x_filesystem: *dataStreamSettings
+    so-metrics-system_x_fsstat: *dataStreamSettings
+    so-metrics-system_x_load: *dataStreamSettings
+    so-metrics-system_x_memory: *dataStreamSettings
+    so-metrics-system_x_network: *dataStreamSettings
+    so-metrics-system_x_ntp: *dataStreamSettings
+    so-metrics-system_x_process: *dataStreamSettings
+    so-metrics-system_x_process_x_summary: *dataStreamSettings
+    so-metrics-system_x_socket_summary: *dataStreamSettings
+    so-metrics-system_x_uptime: *dataStreamSettings
     so-common: *dataStreamSettings
     so-endgame: *dataStreamSettings
     so-idh: *dataStreamSettings
@@ -880,17 +937,6 @@ elasticsearch:
                   global: True
                   advanced: True
                   helpLink: elasticsearch
-              rollover:
-                max_age:
-                  description: Maximum age of index.  Once an index reaches this limit, it will be rolled over into a new index.
-                  global: True
-                  advanced: True
-                  helpLink: elasticsearch
-                max_primary_shard_size:
-                  description: Maximum primary shard size. Once an index reaches this limit, it will be rolled over into a new index.
-                  global: True
-                  advanced: True
-                  helpLink: elasticsearch
               shrink:
                 method:
                   description: Shrink the index to a new index with fewer primary shards. Shrink operation is by count or size.
@@ -987,8 +1033,6 @@ elasticsearch:
             helpLink: elasticsearch
     sos-backup: *indexSettings
     so-detection: *indexSettings
-    so-assistant-chat: *indexSettings
-    so-assistant-session: *indexSettings
     so-metrics-fleet_server_x_agent_status: &fleetMetricsSettings
       index_sorting:
         description: Sorts the index by event time, at the cost of additional processing resource consumption.

salt/elasticsearch/template.map.jinja

@@ -109,9 +109,15 @@
 {%     if not settings.get('index_sorting', False) | to_bool and settings.index_template.template.settings.index.sort is defined %}
 {%       do settings.index_template.template.settings.index.pop('sort') %}
 {%     endif %}
-{%     if DATA_RETENTION_METHOD == 'DLM' and settings.index_template.data_stream is defined and settings.data_stream_lifecycle is defined %}
-{%       if settings.data_stream_lifecycle.data_retention is defined and settings.data_stream_lifecycle.data_retention %}
-{%         do settings.index_template.template.update({'lifecycle': {'data_retention': settings.data_stream_lifecycle.data_retention}}) %}
+{%     if DATA_RETENTION_METHOD == 'DLM' and settings.index_template.data_stream is defined %}
+{#       Addon defaults are generated without data_stream_lifecycle, so fall back to global defaults. #}
+{%       if settings.data_stream_lifecycle is defined %}
+{%         set DATA_STREAM_LIFECYCLE = settings.data_stream_lifecycle %}
+{%       else %}
+{%         set DATA_STREAM_LIFECYCLE = DEFAULT_GLOBAL_OVERRIDES.data_stream_lifecycle %}
+{%       endif %}
+{%       if DATA_STREAM_LIFECYCLE.data_retention is defined and DATA_STREAM_LIFECYCLE.data_retention %}
+{%         do settings.index_template.template.update({'lifecycle': {'data_retention': DATA_STREAM_LIFECYCLE.data_retention}}) %}
 {%       else %}
 {%         do settings.index_template.template.update({'lifecycle': {}}) %}
 {%       endif %}

salt/manager/tools/sbin/soup

@@ -1998,11 +1998,11 @@ main() {
     # Testing that salt-master is up by checking that is it connected to itself
     check_saltmaster_status
 
-    # update the salt-minion configs here and start the minion
+    # update the salt-master and salt-minion configs here and start the minion
     # since highstate are disabled above, minion start should not trigger a highstate
     echo ""
-    echo "Ensuring salt-minion configs are up-to-date."
-    salt-call state.apply salt.minion -l info queue=True
+    echo "Ensuring salt-master and salt-minion configs are up-to-date."
+    salt-call state.apply salt.master -l info queue=True
     echo ""
 
     # ensure the mine is updated and populated before highstates run, following the salt-master restart

salt/salt/master.sls

@@ -94,7 +94,9 @@ salt_master_service:
       - file: checkmine_engine
       - file: pillarWatch_engine
       - file: engines_config
-    - order: 9002
+    - require:
+      - cmd: wait_for_salt_minion_ready
+    - order: last
 
 {% else %}
 

salt/soc/defaults.yaml

@@ -1539,7 +1539,7 @@ soc:
         postgres:
           host: ""
           port: 5432
-          sslMode: "allow"
+          sslMode: "require"
           database: securityonion
           user: ""
           password: ""

salt/soc/files/soc/playbook_placeholder_map.yaml

@@ -14,18 +14,22 @@
 
 CommandLine: process.command_line
 CurrentDirectory: process.working_directory
+Details: registry.data.strings
 Image: process.executable
-ImageLoaded: dll.name
+ImageLoaded: dll.path
 ParentImage: process.parent.executable
 ParentName: process.parent.name
 ParentProcessGuid: process.parent.entity_id
 ProcessGuid: process.entity_id
-TargetFilename: file.name
+TargetFilename: file.path
 TargetObject: registry.path
 TargetUserName: user.target.name
 User: user.name
 community_id: network.community_id
 dns_resolved_ip: dns.resolved_ip
+QueryName: dns.question.name
+dll_basename: dll.name
+file_basename: file.name
 document_id: soc_id
 dst_ip: destination.ip
 dst_port: destination.port
@@ -40,10 +44,15 @@ public_ip: network.public_ip
 related_hosts: related.hosts
 related_ip: related.ip
 src_ip: source.ip
-dns_query_name: dns.query_name
+dns_query_name: dns.question.name
 flow_id: log.id.uid
 payload: network.data.decoded
 rule_category: rule.category
 rule_name: rule.name
 rule_uuid: rule.uuid
 src_port: source.port
+# PowerShell channel (ps_script EID 4104 / classic EID 400)
+ScriptBlockText: powershell.file.script_block_text
+script_block_id: powershell.file.script_block_id
+script_block_hash: powershell.file.script_block_hash
+EngineVersion: powershell.engine.version

salt/soc/files/soc/sigma_playbook_pipeline.yaml

@@ -1,8 +1,9 @@
 name: Security Onion - Playbook Pipeline
 priority: 97
 transformations:
-    # Route string fields to their lowercase-normalized .caseless subfield so wildcard
-    # matches are case-insensitive.
+    # Route to lowercase-normalized .caseless subfields for case-insensitive matching.
+    # file.path.caseless exists on Defend only (Sysmon file events lack it);
+    # registry.path / dll.path / file.name have no .caseless on any source.
     - id: case_insensitive_string_fields
       type: field_name_mapping
       mapping:
@@ -10,3 +11,121 @@ transformations:
         process.parent.executable: process.parent.executable.caseless
         process.command_line: process.command_line.caseless
         process.parent.command_line: process.parent.command_line.caseless
+        file.path: file.path.caseless
+    # file_activity: playbook-only pseudo-category spanning all file operations.
+    - id: playbook_file_activity_add-fields
+      type: add_condition
+      conditions:
+        event.category: 'file'
+      rule_conditions:
+      - type: logsource
+        category: file_activity
+    # EventType -> event.action is safe as a projection only — the values differ
+    # (Sysmon DeleteFile/SetValue vs ECS deletion/modification).
+    - id: playbook_file_activity_event_type
+      type: field_name_mapping
+      mapping:
+        winlog.event_data.EventType: event.action
+      rule_conditions:
+      - type: logsource
+        category: file_activity
+    - id: playbook_file_event_event_type
+      type: field_name_mapping
+      mapping:
+        winlog.event_data.EventType: event.action
+      rule_conditions:
+      - type: logsource
+        category: file_event
+    - id: playbook_file_delete_event_type
+      type: field_name_mapping
+      mapping:
+        winlog.event_data.EventType: event.action
+      rule_conditions:
+      - type: logsource
+        category: file_delete
+    - id: playbook_file_rename_event_type
+      type: field_name_mapping
+      mapping:
+        winlog.event_data.EventType: event.action
+      rule_conditions:
+      - type: logsource
+        category: file_rename
+    - id: playbook_registry_set_event_type
+      type: field_name_mapping
+      mapping:
+        winlog.event_data.EventType: event.action
+      rule_conditions:
+      - type: logsource
+        category: registry_set
+    # WMI-Activity events carry no event.category; event.dataset is source-specific
+    - id: playbook_wmi_activity_add-fields
+      type: add_condition
+      conditions:
+        event.dataset: 'wmi.operational'
+      rule_conditions:
+      - type: logsource
+        category: wmi
+    - id: playbook_wmi_event_activity_add-fields
+      type: add_condition
+      conditions:
+        event.dataset: 'wmi.operational'
+      rule_conditions:
+      - type: logsource
+        category: wmi_event
+    - id: playbook_ps_script_field-mapping
+      type: field_name_mapping
+      mapping:
+        ScriptBlockText: powershell.file.script_block_text
+        Path: file.path
+      rule_conditions:
+      - type: logsource
+        category: ps_script
+    - id: playbook_ps_script_add-fields
+      type: add_condition
+      conditions:
+        event.dataset: 'windows.powershell_operational'
+        event.code: '4104'
+      rule_conditions:
+      - type: logsource
+        category: ps_script
+    # Data (raw EID 400/600 message blob) -> process.command_line: the winlog
+    # integration parses HostApplication into it; `message` is analyzed text and
+    # wildcard matches on it silently return zero. No .caseless on this dataset.
+    - id: playbook_ps_classic_start_field-mapping
+      type: field_name_mapping
+      mapping:
+        Data: process.command_line
+      rule_conditions:
+      - type: logsource
+        category: ps_classic_start
+    - id: playbook_ps_classic_start_add-fields
+      type: add_condition
+      conditions:
+        event.dataset: 'windows.powershell'
+        event.code: '400'
+      rule_conditions:
+      - type: logsource
+        category: ps_classic_start
+    - id: playbook_ps_classic_provider_start_field-mapping
+      type: field_name_mapping
+      mapping:
+        Data: process.command_line
+      rule_conditions:
+      - type: logsource
+        category: ps_classic_provider_start
+    - id: playbook_ps_classic_provider_start_add-fields
+      type: add_condition
+      conditions:
+        event.dataset: 'windows.powershell'
+        event.code: '600'
+      rule_conditions:
+      - type: logsource
+        category: ps_classic_provider_start
+    # Alert docs nest the host under event_data.host.name; no top-level host.name.
+    - id: playbook_alert_host_field
+      type: field_name_mapping
+      mapping:
+        host.name: event_data.host.name
+      rule_conditions:
+      - type: logsource
+        category: alert

salt/soc/files/soc/sigma_so_pipeline.yaml

@@ -131,13 +131,22 @@ transformations:
       type: field_name_mapping
       mapping:
         winlog.event_data.Details: registry.data.strings
-        # field rename only; EventType values (SetValue/CreateKey) still differ from
-        # event.action values (modification/creation)
-        winlog.event_data.EventType: event.action
       rule_conditions:
       - type: logsource
         product: windows
         category: registry_set
+    # ecs_windows renames Initiated -> network.direction without translating the value,
+    # so rules compile to network.direction:"true" and never match (field holds
+    # ingress/egress). network.initiated carries the boolean on both Defend and Sysmon.
+    # Keyed on network.direction because ecs_windows has already renamed by this layer.
+    - id: ecs_fix_network_connection_initiated
+      type: field_name_mapping
+      mapping:
+        network.direction: network.initiated
+      rule_conditions:
+      - type: logsource
+        product: windows
+        category: network_connection
     - id: ecs_fix_image_load
       type: field_name_mapping
       mapping:
@@ -150,6 +159,30 @@ transformations:
       - type: logsource
         product: windows
         category: image_load
+    # Defend reports the driver image in dll.path (file.path is null on driver events) —
+    # same renames as image_load.
+    - id: ecs_fix_driver_load
+      type: field_name_mapping
+      mapping:
+        file.path: dll.path
+        file.code_signature.signed: dll.code_signature.exists
+        winlog.event_data.Signature: dll.code_signature.subject_name
+        file.code_signature.status: dll.code_signature.status
+        winlog.event_data.Hashes: dll.hash.sha256
+      rule_conditions:
+      - type: logsource
+        product: windows
+        category: driver_load
+    - id: ecs_fix_file_rename
+      type: field_name_mapping
+      mapping:
+        # Defend records the pre-rename path in file.Ext.original.path; Sysmon's
+        # SourceFilename has no ECS equivalent.
+        winlog.event_data.SourceFilename: file.Ext.original.path
+      rule_conditions:
+      - type: logsource
+        product: windows
+        category: file_rename
     - id: linux_security_add-fields
       type: add_condition
       conditions:
@@ -323,6 +356,33 @@ transformations:
       rule_conditions:
       - type: logsource
         category: file_event
+    # Without event.type scoping, file_delete rules also match creations.
+    - id: endpoint_file_delete_add-fields
+      type: add_condition
+      conditions:
+        event.category: 'file'
+        event.type: 'deletion'
+      rule_conditions:
+      - type: logsource
+        category: file_delete
+    # Defend reports renames as event.action:rename with event.type:change.
+    - id: endpoint_file_rename_add-fields
+      type: add_condition
+      conditions:
+        event.category: 'file'
+        event.action: 'rename'
+      rule_conditions:
+      - type: logsource
+        category: file_rename
+    # event.type:change selects writes and excludes Defend's read-only registry events.
+    - id: endpoint_registry_set_add-fields
+      type: add_condition
+      conditions:
+        event.category: 'registry'
+        event.type: 'change'
+      rule_conditions:
+      - type: logsource
+        category: registry_set
     # Scope image_load rules to Elastic Endpoint library events (event.category:library, dll.*
     # populated).
     - id: endpoint_image_load_add-fields
@@ -351,6 +411,149 @@ transformations:
       rule_conditions:
       - type: logsource
         category: network_connection
+    # Scope on lookup actions, not network.protocol:dns (also matches Zeek port-53).
+    - id: endpoint_dns_query_add-fields
+      type: add_condition
+      conditions:
+        event.category: 'network'
+        event.action:
+          - 'lookup_requested'
+          - 'lookup_result'
+      rule_conditions:
+      - type: logsource
+        category: dns_query
+    # OS gates: without them a `product: windows` rule in these categories also matches
+    # Linux/macOS. Separate conditions (not `product:` on the mappings above) so the
+    # mappings keep scoping product-less rules.
+    - id: endpoint_file_create_windows_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'windows'
+      rule_conditions:
+      - type: logsource
+        category: file_event
+        product: windows
+    - id: endpoint_file_create_linux_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'linux'
+      rule_conditions:
+      - type: logsource
+        category: file_event
+        product: linux
+    - id: endpoint_file_create_macos_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'macos'
+      rule_conditions:
+      - type: logsource
+        category: file_event
+        product: macos
+    - id: endpoint_file_delete_windows_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'windows'
+      rule_conditions:
+      - type: logsource
+        category: file_delete
+        product: windows
+    - id: endpoint_file_delete_linux_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'linux'
+      rule_conditions:
+      - type: logsource
+        category: file_delete
+        product: linux
+    - id: endpoint_file_delete_macos_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'macos'
+      rule_conditions:
+      - type: logsource
+        category: file_delete
+        product: macos
+    - id: endpoint_file_rename_windows_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'windows'
+      rule_conditions:
+      - type: logsource
+        category: file_rename
+        product: windows
+    - id: endpoint_file_rename_linux_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'linux'
+      rule_conditions:
+      - type: logsource
+        category: file_rename
+        product: linux
+    - id: endpoint_file_rename_macos_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'macos'
+      rule_conditions:
+      - type: logsource
+        category: file_rename
+        product: macos
+    - id: endpoint_network_connection_windows_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'windows'
+      rule_conditions:
+      - type: logsource
+        category: network_connection
+        product: windows
+    - id: endpoint_network_connection_linux_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'linux'
+      rule_conditions:
+      - type: logsource
+        category: network_connection
+        product: linux
+    - id: endpoint_network_connection_macos_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'macos'
+      rule_conditions:
+      - type: logsource
+        category: network_connection
+        product: macos
+    - id: endpoint_dns_query_windows_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'windows'
+      rule_conditions:
+      - type: logsource
+        category: dns_query
+        product: windows
+    - id: endpoint_dns_query_linux_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'linux'
+      rule_conditions:
+      - type: logsource
+        category: dns_query
+        product: linux
+    - id: endpoint_dns_query_macos_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'macos'
+      rule_conditions:
+      - type: logsource
+        category: dns_query
+        product: macos
+    # registry_set / image_load / driver_load are not gated — windows-only telemetry.
+    # wmi / wmi_event are not scoped here: their only handle is event.dataset
+    - id: endpoint_driver_load_add-fields
+      type: add_condition
+      conditions:
+        event.category: 'driver'
+      rule_conditions:
+      - type: logsource
+        category: driver_load
     # Maps "alert" category to SO Alert events
     - id: alert_so_add-fields
       type: add_condition

salt/soc/merged.map.jinja

@@ -88,7 +88,7 @@
           'host': GLOBALS.manager_ip,
           'password': PG_PASS,
           'port': 5432,
-          'sslMode': 'allow',
+          'sslMode': 'require',
           'user': PG_USER,
         }
       }) %}

salt/soc/soc_soc.yaml

@@ -486,7 +486,7 @@ soc:
             global: True
             advanced: True
           sslMode:
-            description: "Use encrypted connections to the PostgreSQL server. Must be one of the following values: disable, allow, prefer, require, verify-ca, verify-full.  Defaults to allow."
+            description: "Use encrypted connections to the PostgreSQL server. Must be one of the following values: disable, allow, prefer, require, verify-ca, verify-full."
             global: True
             advanced: True
           database:

salt/zeek/defaults.yaml

@@ -62,7 +62,6 @@ zeek:
         - securityonion/file-extraction
         - securityonion/community-id-extended
         - oui-logging
-        - icsnpp-modbus
         - icsnpp-dnp3
         - icsnpp-bacnet
         - icsnpp-ethercat