Merge branch 'idstools-refactor' of https://github.com/Security-Onion-Solutions/securityonion into idstools-refactor

FEATURE: Advanced ILM actions via SOC UI

salt/common/tools/sbin_jinja/so-import-pcap

@@ -85,7 +85,7 @@ function suricata() {
   docker run --rm \
     -v /opt/so/conf/suricata/suricata.yaml:/etc/suricata/suricata.yaml:ro \
     -v /opt/so/conf/suricata/threshold.conf:/etc/suricata/threshold.conf:ro \
-    -v /opt/so/conf/suricata/rules:/etc/suricata/rules:ro \
+    -v /opt/so/rules/suricata/:/etc/suricata/rules:ro \
     -v ${LOG_PATH}:/var/log/suricata/:rw \
     -v ${NSM_PATH}/:/nsm/:rw \
     -v "$PCAP:/input.pcap:ro" \

salt/elasticfleet/config.map.jinja

@@ -0,0 +1,34 @@
+{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+   or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+   https://securityonion.net/license; you may not use this file except in compliance with the
+   Elastic License 2.0. #}
+
+{% from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %}
+
+{# advanced config_yaml options for elasticfleet logstash output #}
+{% set ADV_OUTPUT_LOGSTASH_RAW = ELASTICFLEETMERGED.config.outputs.logstash %}
+{% set ADV_OUTPUT_LOGSTASH = {} %}
+{% for k, v in ADV_OUTPUT_LOGSTASH_RAW.items() %}
+{%   if v != "" and v is not none %}
+{%     if k == 'queue_mem_events' %}
+{#       rename queue_mem_events queue.mem.events #}
+{%       do ADV_OUTPUT_LOGSTASH.update({'queue.mem.events':v}) %}
+{%     elif k == 'loadbalance' %}
+{%       if v %}
+{#         only include loadbalance config when its True #}
+{%         do ADV_OUTPUT_LOGSTASH.update({k:v}) %}
+{%       endif %}
+{%     else %}
+{%       do ADV_OUTPUT_LOGSTASH.update({k:v}) %}
+{%     endif %}
+{%   endif %}
+{% endfor %}
+
+{% set LOGSTASH_CONFIG_YAML_RAW = [] %}
+{% if ADV_OUTPUT_LOGSTASH %}
+{%   for k, v in ADV_OUTPUT_LOGSTASH.items() %}
+{%     do LOGSTASH_CONFIG_YAML_RAW.append(k ~ ': ' ~ v) %}
+{%   endfor %}
+{% endif %}
+
+{% set LOGSTASH_CONFIG_YAML = LOGSTASH_CONFIG_YAML_RAW | join('\\n') if LOGSTASH_CONFIG_YAML_RAW else '' %}

salt/elasticfleet/defaults.yaml

@@ -10,6 +10,14 @@ elasticfleet:
       grid_enrollment: ''
     defend_filters:
       enable_auto_configuration: False
+    outputs:
+      logstash:
+        bulk_max_size: ''
+        worker: ''
+        queue_mem_events: ''
+        timeout: ''
+        loadbalance: False
+        compression_level: ''
     subscription_integrations: False
     auto_upgrade_integrations: False
   logging:

salt/elasticfleet/enabled.sls

@@ -32,6 +32,17 @@ so-elastic-fleet-auto-configure-logstash-outputs:
     - retry:
         attempts: 4
         interval: 30
+
+{# Separate from above in order to catch elasticfleet-logstash.crt changes and force update to fleet output policy #}
+so-elastic-fleet-auto-configure-logstash-outputs-force:
+  cmd.run:
+    - name: /usr/sbin/so-elastic-fleet-outputs-update --certs
+    - retry:
+        attempts: 4
+        interval: 30
+    - onchanges:
+        - x509: etc_elasticfleet_logstash_crt
+        - x509: elasticfleet_kafka_crt
 {% endif %}
 
 # If enabled, automatically update Fleet Server URLs & ES Connection

salt/elasticfleet/integration-defaults.map.jinja

@@ -121,6 +121,9 @@
                  "phases": {
                    "cold": {
                      "actions": {
+                        "allocate":{
+                          "number_of_replicas": ""
+                        },
                        "set_priority": {"priority": 0}
                      },
                      "min_age": "60d"
@@ -137,12 +140,31 @@
                          "max_age": "30d",
                          "max_primary_shard_size": "50gb"
                        },
+                       "forcemerge":{
+                         "max_num_segments": ""
+                       },
+                       "shrink":{
+                         "max_primary_shard_size": "",
+                         "method": "COUNT",
+                         "number_of_shards": ""
+                       },
                        "set_priority": {"priority": 100}
                       },
                       "min_age": "0ms"
                    },
                    "warm": {
                      "actions": {
+                        "allocate": {
+                          "number_of_replicas": ""
+                        },
+                        "forcemerge": {
+                          "max_num_segments": ""
+                        },
+                        "shrink":{
+                          "max_primary_shard_size": "",
+                          "method": "COUNT",
+                          "number_of_shards": ""
+                        },
                        "set_priority": {"priority": 50}
                      },
                      "min_age": "30d"

salt/elasticfleet/soc_elasticfleet.yaml

@@ -50,6 +50,46 @@ elasticfleet:
       global: True
       forcedType: bool
       helpLink: elastic-fleet.html
+    outputs:
+      logstash:
+        bulk_max_size:
+          description: The maximum number of events to bulk in a single Logstash request.
+          global: True
+          forcedType: int
+          advanced: True
+          helpLink: elastic-fleet.html
+        worker:
+          description: The number of workers per configured host publishing events.
+          global: True
+          forcedType: int
+          advanced: true
+          helpLink: elastic-fleet.html
+        queue_mem_events:
+          title: queued events
+          description: The number of events the queue can store. This value should be evenly divisible by the smaller of 'bulk_max_size' to avoid sending partial batches to the output.
+          global: True
+          forcedType: int
+          advanced: True
+          helpLink: elastic-fleet.html
+        timeout:
+          description: The number of seconds to wait for responses from the Logstash server before timing out. Eg 30s
+          regex: ^[0-9]+s$
+          advanced: True
+          global: True
+          helpLink: elastic-fleet.html
+        loadbalance:
+          description: If true and multiple Logstash hosts are configured, the output plugin load balances published events onto all Logstash hosts. If false, the output plugin sends all events to one host (determined at random) and switches to another host if the selected one becomes unresponsive.
+          forcedType: bool
+          advanced: True
+          global: True
+          helpLink: elastic-fleet.html
+        compression_level:
+          description: The gzip compression level. The compression level must be in the range of 1 (best speed) to 9 (best compression).
+          regex: ^[1-9]$
+          forcedType: int
+          advanced: True
+          global: True
+          helpLink: elastic-fleet.html
     server:
       custom_fqdn:
         description: Custom FQDN for Agents to connect to. One per line.

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update

@@ -3,11 +3,36 @@
 # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
 # or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use
 # this file except in compliance with the Elastic License 2.0.
-{% from 'vars/globals.map.jinja' import GLOBALS %}
+{%- from 'vars/globals.map.jinja' import GLOBALS %}
-{% from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %}
+{%- from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %}
+{%- from 'elasticfleet/config.map.jinja' import LOGSTASH_CONFIG_YAML %}
 
 . /usr/sbin/so-common
 
+FORCE_UPDATE=false
+UPDATE_CERTS=false
+LOGSTASH_PILLAR_CONFIG_YAML="{{ LOGSTASH_CONFIG_YAML }}"
+LOGSTASH_PILLAR_STATE_FILE="/opt/so/state/esfleet_logstash_config_pillar"
+
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    -f|--force)
+      FORCE_UPDATE=true
+      shift
+      ;;
+    -c| --certs)
+      UPDATE_CERTS=true
+      FORCE_UPDATE=true
+      shift
+      ;;
+    *)
+      echo "Unknown option $1"
+      echo "Usage: $0 [-f|--force] [-c|--certs]"
+      exit 1
+      ;;
+  esac
+done
+
 # Only run on Managers
 if ! is_manager_node; then
     printf "Not a Manager Node... Exiting"
@@ -17,17 +42,49 @@ fi
 function update_logstash_outputs() {
     if logstash_policy=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/outputs/so-manager_logstash" --retry 3 --retry-delay 10 --fail 2>/dev/null); then
         SSL_CONFIG=$(echo "$logstash_policy" | jq -r '.item.ssl')
+        LOGSTASHKEY=$(openssl rsa -in  /etc/pki/elasticfleet-logstash.key)
+        LOGSTASHCRT=$(openssl x509 -in /etc/pki/elasticfleet-logstash.crt)
+        LOGSTASHCA=$(openssl x509 -in  /etc/pki/tls/certs/intca.crt)
+        # Revert escaped \\n to \n for jq
+        LOGSTASH_PILLAR_CONFIG_YAML=$(printf '%b' "$LOGSTASH_PILLAR_CONFIG_YAML")
+
         if SECRETS=$(echo "$logstash_policy" | jq -er '.item.secrets' 2>/dev/null); then
-            JSON_STRING=$(jq -n \
+            if [[ "$UPDATE_CERTS" != "true"  ]]; then
-                --arg UPDATEDLIST "$NEW_LIST_JSON" \
+                # Reuse existing secret
-                --argjson SECRETS "$SECRETS" \
+                JSON_STRING=$(jq -n \
-                --argjson SSL_CONFIG "$SSL_CONFIG" \
+                    --arg UPDATEDLIST "$NEW_LIST_JSON" \
-                '{"name":"grid-logstash","type":"logstash","hosts": $UPDATEDLIST,"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl": $SSL_CONFIG,"secrets": $SECRETS}')
+                    --arg CONFIG_YAML "$LOGSTASH_PILLAR_CONFIG_YAML" \
+                    --argjson SECRETS "$SECRETS" \
+                    --argjson SSL_CONFIG "$SSL_CONFIG" \
+                    '{"name":"grid-logstash","type":"logstash","hosts": $UPDATEDLIST,"is_default":true,"is_default_monitoring":true,"config_yaml":$CONFIG_YAML,"ssl": $SSL_CONFIG,"secrets": $SECRETS}')
+            else
+                # Update certs, creating new secret
+                JSON_STRING=$(jq -n \
+                    --arg UPDATEDLIST "$NEW_LIST_JSON" \
+                    --arg CONFIG_YAML "$LOGSTASH_PILLAR_CONFIG_YAML" \
+                    --arg LOGSTASHKEY "$LOGSTASHKEY" \
+                    --arg LOGSTASHCRT "$LOGSTASHCRT" \
+                    --arg LOGSTASHCA "$LOGSTASHCA" \
+                    '{"name":"grid-logstash","type":"logstash","hosts": $UPDATEDLIST,"is_default":true,"is_default_monitoring":true,"config_yaml":$CONFIG_YAML,"ssl": {"certificate": $LOGSTASHCRT,"certificate_authorities":[ $LOGSTASHCA ]},"secrets": {"ssl":{"key": $LOGSTASHKEY }}}')
+            fi
         else
-            JSON_STRING=$(jq -n \
+            if [[ "$UPDATE_CERTS" != "true" ]]; then
-                --arg UPDATEDLIST "$NEW_LIST_JSON" \
+                # Reuse existing ssl config
-                --argjson SSL_CONFIG "$SSL_CONFIG" \
+                JSON_STRING=$(jq -n \
-                '{"name":"grid-logstash","type":"logstash","hosts": $UPDATEDLIST,"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl": $SSL_CONFIG}')
+                    --arg UPDATEDLIST "$NEW_LIST_JSON" \
+                    --arg CONFIG_YAML "$LOGSTASH_PILLAR_CONFIG_YAML" \
+                    --argjson SSL_CONFIG "$SSL_CONFIG" \
+                    '{"name":"grid-logstash","type":"logstash","hosts": $UPDATEDLIST,"is_default":true,"is_default_monitoring":true,"config_yaml":$CONFIG_YAML,"ssl": $SSL_CONFIG}')
+            else
+                # Update ssl config
+                JSON_STRING=$(jq -n \
+                    --arg UPDATEDLIST "$NEW_LIST_JSON" \
+                    --arg CONFIG_YAML "$LOGSTASH_PILLAR_CONFIG_YAML" \
+                    --arg LOGSTASHKEY "$LOGSTASHKEY" \
+                    --arg LOGSTASHCRT "$LOGSTASHCRT" \
+                    --arg LOGSTASHCA "$LOGSTASHCA" \
+                    '{"name":"grid-logstash","type":"logstash","hosts": $UPDATEDLIST,"is_default":true,"is_default_monitoring":true,"config_yaml":$CONFIG_YAML,"ssl": {"certificate": $LOGSTASHCRT,"key": $LOGSTASHKEY,"certificate_authorities":[ $LOGSTASHCA ]}}')
+            fi
         fi
     fi
 
@@ -38,19 +95,42 @@ function update_kafka_outputs() {
   # Make sure SSL configuration is included in policy updates for Kafka output. SSL is configured in so-elastic-fleet-setup
   if kafka_policy=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/outputs/so-manager_kafka" --fail 2>/dev/null); then
     SSL_CONFIG=$(echo "$kafka_policy" | jq -r '.item.ssl')
+    KAFKAKEY=$(openssl rsa -in  /etc/pki/elasticfleet-kafka.key)
+    KAFKACRT=$(openssl x509 -in /etc/pki/elasticfleet-kafka.crt)
+    KAFKACA=$(openssl x509 -in  /etc/pki/tls/certs/intca.crt)
     if SECRETS=$(echo "$kafka_policy" | jq -er '.item.secrets' 2>/dev/null); then
-      # Update policy when fleet has secrets enabled
+        if [[ "$UPDATE_CERTS" != "true" ]]; then
-      JSON_STRING=$(jq -n \
+            # Update policy when fleet has secrets enabled
-        --arg UPDATEDLIST "$NEW_LIST_JSON" \
+            JSON_STRING=$(jq -n \
-        --argjson SSL_CONFIG "$SSL_CONFIG" \
+              --arg UPDATEDLIST "$NEW_LIST_JSON" \
-        --argjson SECRETS "$SECRETS" \
+              --argjson SSL_CONFIG "$SSL_CONFIG" \
-        '{"name": "grid-kafka","type": "kafka","hosts": $UPDATEDLIST,"is_default": true,"is_default_monitoring": true,"config_yaml": "","ssl": $SSL_CONFIG,"secrets": $SECRETS}')
+              --argjson SECRETS "$SECRETS" \
+              '{"name": "grid-kafka","type": "kafka","hosts": $UPDATEDLIST,"is_default": true,"is_default_monitoring": true,"config_yaml": "","ssl": $SSL_CONFIG,"secrets": $SECRETS}')
+        else
+            # Update certs, creating new secret
+            JSON_STRING=$(jq -n \
+              --arg UPDATEDLIST "$NEW_LIST_JSON" \
+              --arg KAFKAKEY "$KAFKAKEY" \
+              --arg KAFKACRT "$KAFKACRT" \
+              --arg KAFKACA "$KAFKACA" \
+              '{"name": "grid-kafka","type": "kafka","hosts": $UPDATEDLIST,"is_default": true,"is_default_monitoring": true,"config_yaml": "","ssl": {"certificate_authorities":[ $KAFKACA ],"certificate": $KAFKACRT ,"key":"","verification_mode":"full"},"secrets": {"ssl":{"key": $KAFKAKEY }}}')
+        fi
     else
-      # Update policy when fleet has secrets disabled or policy hasn't been force updated
+        if [[ "$UPDATE_CERTS" != "true" ]]; then
-      JSON_STRING=$(jq -n \
+            # Update policy when fleet has secrets disabled or policy hasn't been force updated
-        --arg UPDATEDLIST "$NEW_LIST_JSON" \
+            JSON_STRING=$(jq -n \
-        --argjson SSL_CONFIG "$SSL_CONFIG" \
+              --arg UPDATEDLIST "$NEW_LIST_JSON" \
-        '{"name": "grid-kafka","type": "kafka","hosts": $UPDATEDLIST,"is_default": true,"is_default_monitoring": true,"config_yaml": "","ssl": $SSL_CONFIG}')
+              --argjson SSL_CONFIG "$SSL_CONFIG" \
+              '{"name": "grid-kafka","type": "kafka","hosts": $UPDATEDLIST,"is_default": true,"is_default_monitoring": true,"config_yaml": "","ssl": $SSL_CONFIG}')
+        else
+            # Update ssl config
+            JSON_STRING=$(jq -n \
+              --arg UPDATEDLIST "$NEW_LIST_JSON" \
+              --arg KAFKAKEY "$KAFKAKEY" \
+              --arg KAFKACRT "$KAFKACRT" \
+              --arg KAFKACA "$KAFKACA" \
+              '{"name": "grid-kafka","type": "kafka","hosts": $UPDATEDLIST,"is_default": true,"is_default_monitoring": true,"config_yaml": "","ssl": { "certificate_authorities": [ $KAFKACA ], "certificate": $KAFKACRT, "key": $KAFKAKEY, "verification_mode": "full" }}')
+        fi
     fi
     # Update Kafka outputs
     curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/outputs/so-manager_kafka" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" | jq
@@ -73,7 +153,7 @@ function update_kafka_outputs() {
 
   # Get the current list of kafka outputs & hash them
   CURRENT_LIST=$(jq -c -r '.item.hosts' <<<  "$RAW_JSON")
-  CURRENT_HASH=$(sha1sum <<< "$CURRENT_LIST" | awk '{print $1}')
+  CURRENT_HASH=$(sha256sum <<< "$CURRENT_LIST" | awk '{print $1}')
 
   declare -a NEW_LIST=()
 
@@ -96,10 +176,19 @@ function update_kafka_outputs() {
    printf "Failed to query for current Logstash Outputs..."
    exit 1
   fi
+  # logstash adv config - compare pillar to last state file value
+  if [[ -f "$LOGSTASH_PILLAR_STATE_FILE" ]]; then
+    PREVIOUS_LOGSTASH_PILLAR_CONFIG_YAML=$(cat "$LOGSTASH_PILLAR_STATE_FILE")
+    if [[ "$LOGSTASH_PILLAR_CONFIG_YAML" != "$PREVIOUS_LOGSTASH_PILLAR_CONFIG_YAML" ]]; then
+      echo "Logstash pillar config has changed - forcing update"
+      FORCE_UPDATE=true
+    fi
+    echo "$LOGSTASH_PILLAR_CONFIG_YAML" > "$LOGSTASH_PILLAR_STATE_FILE"
+  fi
 
   # Get the current list of Logstash outputs & hash them
   CURRENT_LIST=$(jq -c -r '.item.hosts' <<<  "$RAW_JSON")
-  CURRENT_HASH=$(sha1sum <<< "$CURRENT_LIST" | awk '{print $1}')
+  CURRENT_HASH=$(sha256sum <<< "$CURRENT_LIST" | awk '{print $1}')
 
   declare -a NEW_LIST=()
 
@@ -148,10 +237,10 @@ function update_kafka_outputs() {
 
 # Sort & hash the new list of Logstash Outputs
 NEW_LIST_JSON=$(jq --compact-output --null-input '$ARGS.positional' --args -- "${NEW_LIST[@]}")
-NEW_HASH=$(sha1sum <<< "$NEW_LIST_JSON" | awk '{print $1}')
+NEW_HASH=$(sha256sum <<< "$NEW_LIST_JSON" | awk '{print $1}')
 
 # Compare the current & new list of outputs - if different, update the Logstash outputs
-if [ "$NEW_HASH" = "$CURRENT_HASH" ]; then
+if [[ "$NEW_HASH" = "$CURRENT_HASH" ]] && [[ "$FORCE_UPDATE" != "true" ]]; then
     printf "\nHashes match - no update needed.\n"
     printf "Current List: $CURRENT_LIST\nNew List: $NEW_LIST_JSON\n"
 

salt/elasticsearch/defaults.yaml

@@ -72,6 +72,8 @@ elasticsearch:
             actions:
               set_priority:
                 priority: 0
+              allocate:
+                number_of_replicas: ""
             min_age: 60d
           delete:
             actions:
@@ -84,11 +86,25 @@ elasticsearch:
                 max_primary_shard_size: 50gb
               set_priority:
                 priority: 100
+              forcemerge:
+                max_num_segments: ""
+              shrink:
+                max_primary_shard_size: ""
+                method: COUNT
+                number_of_shards: ""
             min_age: 0ms
           warm:
             actions:
               set_priority:
                 priority: 50
+              forcemerge:
+                max_num_segments: ""
+              shrink:
+                max_primary_shard_size: ""
+                method: COUNT
+                number_of_shards: ""
+              allocate:
+                number_of_replicas: ""
             min_age: 30d
     so-case:
       index_sorting: false
@@ -245,7 +261,6 @@ elasticsearch:
               set_priority:
                 priority: 50
             min_age: 30d
-      warm: 7
     so-detection:
       index_sorting: false
       index_template:
@@ -584,7 +599,6 @@ elasticsearch:
               set_priority:
                 priority: 50
             min_age: 30d
-      warm: 7
     so-import:
       index_sorting: false
       index_template:
@@ -932,7 +946,6 @@ elasticsearch:
               set_priority:
                 priority: 50
             min_age: 30d
-      warm: 7
     so-hydra:
       close: 30
       delete: 365
@@ -1043,7 +1056,6 @@ elasticsearch:
               set_priority:
                 priority: 50
             min_age: 30d
-      warm: 7
     so-lists:
       index_sorting: false
       index_template:
@@ -1127,6 +1139,8 @@ elasticsearch:
             actions:
               set_priority:
                 priority: 0
+              allocate:
+                number_of_replicas: ""
             min_age: 60d
           delete:
             actions:
@@ -1139,11 +1153,25 @@ elasticsearch:
                 max_primary_shard_size: 50gb
               set_priority:
                 priority: 100
+              forcemerge:
+                max_num_segments: ""
+              shrink:
+                max_primary_shard_size: ""
+                method: COUNT
+                number_of_shards: ""
             min_age: 0ms
           warm:
             actions:
               set_priority:
                 priority: 50
+              allocate:
+                number_of_replicas: ""
+              forcemerge:
+                max_num_segments: ""
+              shrink:
+                max_primary_shard_size: ""
+                method: COUNT
+                number_of_shards: ""
             min_age: 30d
     so-logs-detections_x_alerts:
       index_sorting: false
@@ -3123,7 +3151,6 @@ elasticsearch:
               set_priority:
                 priority: 50
             min_age: 30d
-      warm: 7
     so-logs-system_x_application:
       index_sorting: false
       index_template:

salt/elasticsearch/soc_elasticsearch.yaml

@@ -131,6 +131,47 @@ elasticsearch:
                   description: Maximum primary shard size. Once an index reaches this limit, it will be rolled over into a new index.
                   global: True
                   helpLink: elasticsearch.html
+              shrink:
+                method:
+                  description: Shrink the index to a new index with fewer primary shards. Shrink operation is by count or size.
+                  options:
+                  - COUNT
+                  - SIZE
+                  global: True
+                  advanced: True
+                  forcedType: string
+                number_of_shards:
+                  title: shard count
+                  description: Desired shard count. Note that this value is only used when the shrink method selected is 'COUNT'.
+                  global: True
+                  forcedType: int
+                  advanced: True
+                max_primary_shard_size:
+                  title: max shard size
+                  description: Desired shard size in gb/tb/pb eg. 100gb. Note that this value is only used when the shrink method selected is 'SIZE'.
+                  regex: ^[0-9]+(?:gb|tb|pb)$
+                  global: True
+                  forcedType: string
+                  advanced: True
+                allow_write_after_shrink:
+                  description: Allow writes after shrink.
+                  global: True
+                  forcedType: bool
+                  default: False
+                  advanced: True
+              forcemerge:
+                max_num_segments:
+                  description: Reduce the number of segments in each index shard and clean up deleted documents.
+                  global: True
+                  forcedType: int
+                  advanced: True
+                index_codec:
+                  title: compression
+                  description: Use higher compression for stored fields at the cost of slower performance.
+                  forcedType: bool
+                  global: True
+                  default: False
+                  advanced: True
           cold:
             min_age:
               description: Minimum age of index. ex. 60d - This determines when the index should be moved to the cold tier. While still searchable, this tier is typically optimized for lower storage costs rather than search speed. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and cold min_age set to 60 then there will be 30 days from index creation to rollover and then an additional 60 days before moving to cold tier.
@@ -144,6 +185,12 @@ elasticsearch:
                   description: Used for index recovery after a node restart. Indices with higher priorities are recovered before indices with lower priorities.
                   global: True
                   helpLink: elasticsearch.html
+              allocate:
+                number_of_replicas:
+                  description: Set the number of replicas. Remains the same as the previous phase by default.
+                  forcedType: int
+                  global: True
+                  advanced: True
           warm:
             min_age: 
               description: Minimum age of index. ex. 30d - This determines when the index should be moved to the warm tier. Nodes in the warm tier generally don’t need to be as fast as those in the hot tier. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and warm min_age set to 30 then there will be 30 days from index creation to rollover and then an additional 30 days before moving to warm tier.
@@ -158,6 +205,52 @@ elasticsearch:
                   forcedType: int
                   global: True
                   helpLink: elasticsearch.html
+              shrink:
+                method:
+                  description: Shrink the index to a new index with fewer primary shards. Shrink operation is by count or size.
+                  options:
+                  - COUNT
+                  - SIZE
+                  global: True
+                  advanced: True
+                number_of_shards:
+                  title: shard count
+                  description: Desired shard count. Note that this value is only used when the shrink method selected is 'COUNT'.
+                  global: True
+                  forcedType: int
+                  advanced: True
+                max_primary_shard_size:
+                  title: max shard size
+                  description: Desired shard size in gb/tb/pb eg. 100gb. Note that this value is only used when the shrink method selected is 'SIZE'.
+                  regex: ^[0-9]+(?:gb|tb|pb)$
+                  global: True
+                  forcedType: string
+                  advanced: True
+                allow_write_after_shrink:
+                  description: Allow writes after shrink.
+                  global: True
+                  forcedType: bool
+                  default: False
+                  advanced: True
+              forcemerge:
+                max_num_segments:
+                  description: Reduce the number of segments in each index shard and clean up deleted documents.
+                  global: True
+                  forcedType: int
+                  advanced: True
+                index_codec:
+                  title: compression
+                  description: Use higher compression for stored fields at the cost of slower performance.
+                  forcedType: bool
+                  global: True
+                  default: False
+                  advanced: True
+              allocate:
+                number_of_replicas:
+                  description: Set the number of replicas. Remains the same as the previous phase by default.
+                  forcedType: int
+                  global: True
+                  advanced: True
           delete:
             min_age:
               description: Minimum age of index. ex. 90d - This determines when the index should be deleted. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and delete min_age set to 90 then there will be 30 days from index creation to rollover and then an additional 90 days before deletion.
@@ -287,6 +380,47 @@ elasticsearch:
                   global: True
                   advanced: True
                   helpLink: elasticsearch.html
+              shrink:
+                method:
+                  description: Shrink the index to a new index with fewer primary shards. Shrink operation is by count or size.
+                  options:
+                  - COUNT
+                  - SIZE
+                  global: True
+                  advanced: True
+                  forcedType: string
+                number_of_shards:
+                  title: shard count
+                  description: Desired shard count. Note that this value is only used when the shrink method selected is 'COUNT'.
+                  global: True
+                  forcedType: int
+                  advanced: True
+                max_primary_shard_size:
+                  title: max shard size
+                  description: Desired shard size in gb/tb/pb eg. 100gb. Note that this value is only used when the shrink method selected is 'SIZE'.
+                  regex: ^[0-9]+(?:gb|tb|pb)$
+                  global: True
+                  forcedType: string
+                  advanced: True
+                allow_write_after_shrink:
+                  description: Allow writes after shrink.
+                  global: True
+                  forcedType: bool
+                  default: False
+                  advanced: True
+              forcemerge:
+                max_num_segments:
+                  description: Reduce the number of segments in each index shard and clean up deleted documents.
+                  global: True
+                  forcedType: int
+                  advanced: True
+                index_codec:
+                  title: compression
+                  description: Use higher compression for stored fields at the cost of slower performance.
+                  forcedType: bool
+                  global: True
+                  default: False
+                  advanced: True
           warm:
             min_age:
               description: Minimum age of index. ex. 30d - This determines when the index should be moved to the warm tier. Nodes in the warm tier generally don’t need to be as fast as those in the hot tier. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and warm min_age set to 30 then there will be 30 days from index creation to rollover and then an additional 30 days before moving to warm tier.
@@ -314,6 +448,52 @@ elasticsearch:
                   global: True
                   advanced: True
                   helpLink: elasticsearch.html
+              shrink:
+                method:
+                  description: Shrink the index to a new index with fewer primary shards. Shrink operation is by count or size.
+                  options:
+                  - COUNT
+                  - SIZE
+                  global: True
+                  advanced: True
+                number_of_shards:
+                  title: shard count
+                  description: Desired shard count. Note that this value is only used when the shrink method selected is 'COUNT'.
+                  global: True
+                  forcedType: int
+                  advanced: True
+                max_primary_shard_size:
+                  title: max shard size
+                  description: Desired shard size in gb/tb/pb eg. 100gb. Note that this value is only used when the shrink method selected is 'SIZE'.
+                  regex: ^[0-9]+(?:gb|tb|pb)$
+                  global: True
+                  forcedType: string
+                  advanced: True
+                allow_write_after_shrink:
+                  description: Allow writes after shrink.
+                  global: True
+                  forcedType: bool
+                  default: False
+                  advanced: True
+              forcemerge:
+                max_num_segments:
+                  description: Reduce the number of segments in each index shard and clean up deleted documents.
+                  global: True
+                  forcedType: int
+                  advanced: True
+                index_codec:
+                  title: compression
+                  description: Use higher compression for stored fields at the cost of slower performance.
+                  forcedType: bool
+                  global: True
+                  default: False
+                  advanced: True
+              allocate:
+                number_of_replicas:
+                  description: Set the number of replicas. Remains the same as the previous phase by default.
+                  forcedType: int
+                  global: True
+                  advanced: True
           cold:
             min_age:
               description: Minimum age of index. ex. 60d - This determines when the index should be moved to the cold tier.  While still searchable, this tier is typically optimized for lower storage costs rather than search speed. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and cold min_age set to 60 then there will be 30 days from index creation to rollover and then an additional 60 days before moving to cold tier.
@@ -330,6 +510,12 @@ elasticsearch:
                   global: True
                   advanced: True
                   helpLink: elasticsearch.html
+              allocate:
+                number_of_replicas:
+                  description: Set the number of replicas. Remains the same as the previous phase by default.
+                  forcedType: int
+                  global: True
+                  advanced: True
           delete:
             min_age:
               description: Minimum age of index. ex. 90d - This determines when the index should be deleted. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and delete min_age set to 90 then there will be 30 days from index creation to rollover and then an additional 90 days before deletion.

salt/elasticsearch/template.map.jinja

@@ -61,5 +61,55 @@
 {%       do settings.index_template.template.settings.index.pop('sort') %}
 {%     endif %}
 {%   endif %}
+
+{# advanced ilm actions #}
+{%   if settings.policy is defined and settings.policy.phases is defined %}
+{%     set PHASE_NAMES = ["hot", "warm", "cold"] %}
+{%     for P in PHASE_NAMES %}
+{%       if settings.policy.phases[P] is defined and settings.policy.phases[P].actions is defined %}
+{%         set PHASE = settings.policy.phases[P].actions %}
+{#         remove allocate action if number_of_replicas isn't configured #}
+{%         if PHASE.allocate is defined %}
+{%           if PHASE.allocate.number_of_replicas is not defined or PHASE.allocate.number_of_replicas == "" %}
+{%             do PHASE.pop('allocate', none) %}
+{%           endif %}
+{%         endif %}
+{#         start shrink action #}
+{%         if PHASE.shrink is defined %}
+{%           if PHASE.shrink.method is defined %}
+{%             if PHASE.shrink.method == 'COUNT' and PHASE.shrink.number_of_shards is defined and PHASE.shrink.number_of_shards %}
+{#               remove max_primary_shard_size value when doing shrink operation by count vs size #}
+{%               do PHASE.shrink.pop('max_primary_shard_size', none) %}
+{%             elif PHASE.shrink.method == 'SIZE' and PHASE.shrink.max_primary_shard_size is defined and PHASE.shrink.max_primary_shard_size %}
+{#               remove number_of_shards value when doing shrink operation by size vs count #}
+{%               do PHASE.shrink.pop('number_of_shards', none) %}
+{%             else %}
+{#               method isn't defined or missing a required config number_of_shards/max_primary_shard_size #}
+{%               do PHASE.pop('shrink', none) %}
+{%             endif %}
+{%           endif %}
+{%         endif %}
+{#         always remove shrink method since its only used for SOC config, not in the actual ilm policy #}
+{%         if PHASE.shrink is defined %}
+{%            do PHASE.shrink.pop('method', none) %}
+{%         endif %}
+{#         end shrink action #}
+{#         start force merge #}
+{%         if PHASE.forcemerge is defined %}
+{%           if PHASE.forcemerge.index_codec is defined and PHASE.forcemerge.index_codec %}
+{%             do PHASE.forcemerge.update({'index_codec': 'best_compression'}) %}
+{%           else %}
+{%             do PHASE.forcemerge.pop('index_codec', none) %}
+{%           endif %}
+{%           if PHASE.forcemerge.max_num_segments is not defined or not PHASE.forcemerge.max_num_segments %}
+{#             max_num_segments is empty, drop it #}
+{%             do PHASE.pop('forcemerge', none) %}
+{%           endif %}
+{%         endif %}
+{#         end force merge #}
+{%       endif %}
+{%     endfor %}
+{%   endif %}
+
 {%   do ES_INDEX_SETTINGS.update({index | replace("_x_", "."): ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index]}) %}
 {% endfor %}

salt/manager/managed_soc_annotations.sls

@@ -25,13 +25,11 @@
            {% set index_settings = es.get('index_settings', {}) %}
            {% set input = index_settings.get('so-logs', {}) %}
            {% for k in matched_integration_names %}
-           {%   if k not in index_settings %}
+           {%   do index_settings.update({k: input}) %}
-           {%     set _ = index_settings.update({k: input}) %}
-           {%   endif %}
            {% endfor %}
            {% for k in addon_integration_keys %}
            {%   if k not in matched_integration_names and k in index_settings %}
-           {%     set _ = index_settings.pop(k) %}
+           {%     do index_settings.pop(k) %}
            {%   endif %}
            {% endfor %}
            {{ data }}
@@ -45,14 +43,12 @@
            {% set es = data.get('elasticsearch', {}) %}
            {% set index_settings = es.get('index_settings', {}) %}
            {% for k in matched_integration_names %}
-           {%   if k not in index_settings %}
+           {%   set input = ADDON_INTEGRATION_DEFAULTS[k] %}
-           {%     set input = ADDON_INTEGRATION_DEFAULTS[k] %}
+           {%     do index_settings.update({k: input})%}
-           {%     set _ = index_settings.update({k: input})%}
-           {%   endif %}
            {% endfor %}
            {% for k in addon_integration_keys %}
            {%   if k not in matched_integration_names and k in index_settings %}
-           {%     set _ = index_settings.pop(k) %}
+           {%     do index_settings.pop(k) %}
            {%   endif %}
            {% endfor %}
            {{ data }}

salt/manager/tools/sbin/so-yaml.py

@@ -17,6 +17,7 @@ def showUsage(args):
     print('Usage: {} <COMMAND> <YAML_FILE> [ARGS...]'.format(sys.argv[0]), file=sys.stderr)
     print('  General commands:', file=sys.stderr)
     print('    append         - Append a list item to a yaml key, if it exists and is a list. Requires KEY and LISTITEM args.', file=sys.stderr)
+    print('    removelistitem - Remove a list item from a yaml key, if it exists and is a list. Requires KEY and LISTITEM args.', file=sys.stderr)
     print('    add            - Add a new key and set its value. Fails if key already exists. Requires KEY and VALUE args.', file=sys.stderr)
     print('    get            - Displays (to stdout) the value stored in the given key. Requires KEY arg.', file=sys.stderr)
     print('    remove         - Removes a yaml key, if it exists. Requires KEY arg.', file=sys.stderr)
@@ -57,6 +58,24 @@ def appendItem(content, key, listItem):
             return 1
 
 
+def removeListItem(content, key, listItem):
+    pieces = key.split(".", 1)
+    if len(pieces) > 1:
+        removeListItem(content[pieces[0]], pieces[1], listItem)
+    else:
+        try:
+            if not isinstance(content[key], list):
+                raise AttributeError("Value is not a list")
+            if listItem in content[key]:
+                content[key].remove(listItem)
+        except (AttributeError, TypeError):
+            print("The existing value for the given key is not a list. No action was taken on the file.", file=sys.stderr)
+            return 1
+        except KeyError:
+            print("The key provided does not exist. No action was taken on the file.", file=sys.stderr)
+            return 1
+
+
 def convertType(value):
     if isinstance(value, str) and value.startswith("file:"):
         path = value[5:]  # Remove "file:" prefix
@@ -103,6 +122,23 @@ def append(args):
     return 0
 
 
+def removelistitem(args):
+    if len(args) != 3:
+        print('Missing filename, key arg, or list item to remove', file=sys.stderr)
+        showUsage(None)
+        return 1
+
+    filename = args[0]
+    key = args[1]
+    listItem = args[2]
+
+    content = loadYaml(filename)
+    removeListItem(content, key, convertType(listItem))
+    writeYaml(filename, content)
+
+    return 0
+
+
 def addKey(content, key, value):
     pieces = key.split(".", 1)
     if len(pieces) > 1:
@@ -211,6 +247,7 @@ def main():
         "help": showUsage,
         "add": add,
         "append": append,
+        "removelistitem": removelistitem,
         "get": get,
         "remove": remove,
         "replace": replace,

salt/manager/tools/sbin/so-yaml_test.py

@@ -457,3 +457,126 @@ class TestRemove(unittest.TestCase):
                 self.assertEqual(result, 1)
                 self.assertIn("Missing filename or key arg", mock_stderr.getvalue())
                 sysmock.assert_called_once_with(1)
+
+
+class TestRemoveListItem(unittest.TestCase):
+
+    def test_removelistitem_missing_arg(self):
+        with patch('sys.exit', new=MagicMock()) as sysmock:
+            with patch('sys.stderr', new=StringIO()) as mock_stderr:
+                sys.argv = ["cmd", "help"]
+                soyaml.removelistitem(["file", "key"])
+                sysmock.assert_called()
+                self.assertIn("Missing filename, key arg, or list item to remove", mock_stderr.getvalue())
+
+    def test_removelistitem(self):
+        filename = "/tmp/so-yaml_test-removelistitem.yaml"
+        file = open(filename, "w")
+        file.write("{key1: { child1: 123, child2: abc }, key2: false, key3: [a,b,c]}")
+        file.close()
+
+        soyaml.removelistitem([filename, "key3", "b"])
+
+        file = open(filename, "r")
+        actual = file.read()
+        file.close()
+
+        expected = "key1:\n  child1: 123\n  child2: abc\nkey2: false\nkey3:\n- a\n- c\n"
+        self.assertEqual(actual, expected)
+
+    def test_removelistitem_nested(self):
+        filename = "/tmp/so-yaml_test-removelistitem.yaml"
+        file = open(filename, "w")
+        file.write("{key1: { child1: 123, child2: [a,b,c] }, key2: false, key3: [e,f,g]}")
+        file.close()
+
+        soyaml.removelistitem([filename, "key1.child2", "b"])
+
+        file = open(filename, "r")
+        actual = file.read()
+        file.close()
+
+        expected = "key1:\n  child1: 123\n  child2:\n  - a\n  - c\nkey2: false\nkey3:\n- e\n- f\n- g\n"
+        self.assertEqual(actual, expected)
+
+    def test_removelistitem_nested_deep(self):
+        filename = "/tmp/so-yaml_test-removelistitem.yaml"
+        file = open(filename, "w")
+        file.write("{key1: { child1: 123, child2: { deep1: 45, deep2: [a,b,c] } }, key2: false, key3: [e,f,g]}")
+        file.close()
+
+        soyaml.removelistitem([filename, "key1.child2.deep2", "b"])
+
+        file = open(filename, "r")
+        actual = file.read()
+        file.close()
+
+        expected = "key1:\n  child1: 123\n  child2:\n    deep1: 45\n    deep2:\n    - a\n    - c\nkey2: false\nkey3:\n- e\n- f\n- g\n"
+        self.assertEqual(actual, expected)
+
+    def test_removelistitem_item_not_in_list(self):
+        filename = "/tmp/so-yaml_test-removelistitem.yaml"
+        file = open(filename, "w")
+        file.write("{key1: [a,b,c]}")
+        file.close()
+
+        soyaml.removelistitem([filename, "key1", "d"])
+
+        file = open(filename, "r")
+        actual = file.read()
+        file.close()
+
+        expected = "key1:\n- a\n- b\n- c\n"
+        self.assertEqual(actual, expected)
+
+    def test_removelistitem_key_noexist(self):
+        filename = "/tmp/so-yaml_test-removelistitem.yaml"
+        file = open(filename, "w")
+        file.write("{key1: { child1: 123, child2: { deep1: 45, deep2: [a,b,c] } }, key2: false, key3: [e,f,g]}")
+        file.close()
+
+        with patch('sys.exit', new=MagicMock()) as sysmock:
+            with patch('sys.stderr', new=StringIO()) as mock_stderr:
+                sys.argv = ["cmd", "removelistitem", filename, "key4", "h"]
+                soyaml.main()
+                sysmock.assert_called()
+                self.assertEqual("The key provided does not exist. No action was taken on the file.\n", mock_stderr.getvalue())
+
+    def test_removelistitem_key_noexist_deep(self):
+        filename = "/tmp/so-yaml_test-removelistitem.yaml"
+        file = open(filename, "w")
+        file.write("{key1: { child1: 123, child2: { deep1: 45, deep2: [a,b,c] } }, key2: false, key3: [e,f,g]}")
+        file.close()
+
+        with patch('sys.exit', new=MagicMock()) as sysmock:
+            with patch('sys.stderr', new=StringIO()) as mock_stderr:
+                sys.argv = ["cmd", "removelistitem", filename, "key1.child2.deep3", "h"]
+                soyaml.main()
+                sysmock.assert_called()
+                self.assertEqual("The key provided does not exist. No action was taken on the file.\n", mock_stderr.getvalue())
+
+    def test_removelistitem_key_nonlist(self):
+        filename = "/tmp/so-yaml_test-removelistitem.yaml"
+        file = open(filename, "w")
+        file.write("{key1: { child1: 123, child2: { deep1: 45, deep2: [a,b,c] } }, key2: false, key3: [e,f,g]}")
+        file.close()
+
+        with patch('sys.exit', new=MagicMock()) as sysmock:
+            with patch('sys.stderr', new=StringIO()) as mock_stderr:
+                sys.argv = ["cmd", "removelistitem", filename, "key1", "h"]
+                soyaml.main()
+                sysmock.assert_called()
+                self.assertEqual("The existing value for the given key is not a list. No action was taken on the file.\n", mock_stderr.getvalue())
+
+    def test_removelistitem_key_nonlist_deep(self):
+        filename = "/tmp/so-yaml_test-removelistitem.yaml"
+        file = open(filename, "w")
+        file.write("{key1: { child1: 123, child2: { deep1: 45, deep2: [a,b,c] } }, key2: false, key3: [e,f,g]}")
+        file.close()
+
+        with patch('sys.exit', new=MagicMock()) as sysmock:
+            with patch('sys.stderr', new=StringIO()) as mock_stderr:
+                sys.argv = ["cmd", "removelistitem", filename, "key1.child2.deep1", "h"]
+                soyaml.main()
+                sysmock.assert_called()
+                self.assertEqual("The existing value for the given key is not a list. No action was taken on the file.\n", mock_stderr.getvalue())

salt/manager/tools/sbin/soup

@@ -916,6 +916,8 @@ up_to_2.4.200() {
   echo "Backing up idstools config..."
   suricata_idstools_removal_pre
 
+  touch /opt/so/state/esfleet_logstash_config_pillar
+
   INSTALLEDVERSION=2.4.200
 }
 
@@ -1179,14 +1181,39 @@ hash_normalized_file() {
         "$file" | sha256sum | awk '{print $1}'
 }
 
-# Known-default hashes
+# Known-default hashes for so-rule-update (ETOPEN ruleset)
 KNOWN_SO_RULE_UPDATE_HASHES=(
-    "8f1fe1cb65c08aab78830315b952785c7ccdcc108c5c0474f427e29d4e39ee5f"  # non-Airgap
+    # 2.4.100+ (suricata 7.0.3, non-airgap)
-    "d23ac5a962c709dcb888103effb71444df72b46009b6c426e280dbfbc7d74d40"  # Airgap
+    "5fbd067ced86c8ec72ffb7e1798aa624123b536fb9d78f4b3ad8d3b45db1eae7"  # 2.4.100-2.4.190 non-Airgap
+    # 2.4.90+ airgap (same for 2.4.90 and 2.4.100+)
+    "61f632c55791338c438c071040f1490066769bcce808b595b5cc7974a90e653a"  # 2.4.90+ Airgap
+    # 2.4.90 (suricata 6.0, non-airgap, comment inside proxy block)
+    "0380ec52a05933244ab0f0bc506576e1d838483647b40612d5fe4b378e47aedd"  # 2.4.90 non-Airgap
+    # 2.4.10-2.4.80 (suricata 6.0, non-airgap, comment outside proxy block)
+    "b6e4d1b5a78d57880ad038a9cd2cc6978aeb2dd27d48ea1a44dd866a2aee7ff4"  # 2.4.10-2.4.80 non-Airgap
+    # 2.4.10-2.4.80 airgap
+    "b20146526ace2b142fde4664f1386a9a1defa319b3a1d113600ad33a1b037dad"  # 2.4.10-2.4.80 Airgap
+    # 2.4.5 and earlier (no pidof check, non-airgap)
+    "d04f5e4015c348133d28a7840839e82d60009781eaaa1c66f7f67747703590dc"  # 2.4.5 non-Airgap
 )
 
+# Known-default hashes for rulecat.conf
 KNOWN_RULECAT_CONF_HASHES=(
-    "17fc663a83b30d4ba43ac6643666b0c96343c5ea6ea833fe6a8362fe415b666b"  # default
+    # 2.4.100+ (suricata 7.0.3)
+    "302e75dca9110807f09ade2eec3be1fcfc8b2bf6cf2252b0269bb72efeefe67e"  # 2.4.100-2.4.190 without SURICATA md_engine
+    "8029b7718c324a9afa06a5cf180afde703da1277af4bdd30310a6cfa3d6398cb"  # 2.4.100-2.4.190 with SURICATA md_engine
+    # 2.4.80-2.4.90 (suricata 6.0, with --suricata-version and --output)
+    "4d8b318e6950a6f60b02f307cf27c929efd39652990c1bd0c8820aa8a307e1e7"  # 2.4.80-2.4.90 without SURICATA md_engine
+    "a1ddf264c86c4e91c81c5a317f745a19466d4311e4533ec3a3c91fed04c11678"  # 2.4.80-2.4.90 with SURICATA md_engine
+    # 2.4.50-2.4.70 (/suri/ path, no --suricata-version)
+    "86e3afb8d0f00c62337195602636864c98580a13ca9cc85029661a539deae6ae"  # 2.4.50-2.4.70 without SURICATA md_engine
+    "5a97604ca5b820a10273a2d6546bb5e00c5122ca5a7dfe0ba0bfbce5fc026f4b"  # 2.4.50-2.4.70 with SURICATA md_engine
+    # 2.4.20-2.4.40 (/nids/ path without /suri/)
+    "d098ea9ecd94b5cca35bf33543f8ea8f48066a0785221fabda7fef43d2462c29"  # 2.4.20-2.4.40 without SURICATA md_engine
+    "9dbc60df22ae20d65738ba42e620392577857038ba92278e23ec182081d191cd"  # 2.4.20-2.4.40 with SURICATA md_engine
+    # 2.4.5-2.4.10 (/sorules/ path for extraction/filters)
+    "490f6843d9fca759ee74db3ada9c702e2440b8393f2cfaf07bbe41aaa6d955c3"  # 2.4.5-2.4.10 with SURICATA md_engine
+    # Note: 2.4.5-2.4.10 without SURICATA md_engine has same hash as 2.4.20-2.4.40 without SURICATA md_engine
 )
 
 # Check a config file against known hashes
@@ -1255,6 +1282,31 @@ else
     echo "Custom idstools configuration detected - syncBlock remains in place"
     echo "Review /opt/so/conf/soc/fingerprints/suricataengine.syncBlock for details"
 fi
+
+echo "Cleaning up idstools"
+echo "Stopping and removing the idstools container..."
+if [ -n "$(docker ps -q -f name=^so-idstools$)" ]; then
+    image_name=$(docker ps -a --filter name=^so-idstools$ --format '{{.Image}}' 2>/dev/null || true)
+    docker stop so-idstools || echo "Warning: failed to stop so-idstools container"
+    docker rm so-idstools || echo "Warning: failed to remove so-idstools container"
+
+    if [[ -n "$image_name" ]]; then
+        echo "Removing idstools image: $image_name"
+        docker rmi "$image_name" || echo "Warning: failed to remove image $image_name"
+    fi
+fi
+
+echo "Removing idstools symlink and scripts..."
+rm /opt/so/saltstack/local/salt/suricata/rules
+rm -rf /usr/sbin/so-idstools*
+sed -i '/^#\?so-idstools$/d' /opt/so/conf/so-status/so-status.conf
+
+# Backup the salt master config & manager pillar before editing it
+cp /opt/so/saltstack/local/pillar/minions/$MINIONID.sls /nsm/backup/detections-migration/2-4-200/
+cp /etc/salt/master  /nsm/backup/detections-migration/2-4-200/
+so-yaml.py remove /opt/so/saltstack/local/pillar/minions/$MINIONID.sls idstools
+so-yaml.py removelistitem /etc/salt/master file_roots.base /opt/so/rules/nids
+
 }
 
 determine_elastic_agent_upgrade() {

salt/suricata/config.sls

@@ -22,6 +22,14 @@ suriPCAPbpfcompilationfailure:
 {%     endif %}
 {%   endif %}
 
+suridir:
+  file.directory:
+    - name: /opt/so/conf/suricata
+    - user: 940
+    - group: 939
+    - mode: 775
+    - makedirs: True
+
 # BPF applied to all of Suricata - alerts/metadata/pcap
 suribpf:
   file.managed:
@@ -81,13 +89,6 @@ suricata_sbin_jinja:
     - file_mode: 755
     - template: jinja
 
-suridir:
-  file.directory:
-    - name: /opt/so/conf/suricata
-    - user: 940
-    - group: 939
-    - mode: 775
-
 suriruledir:
   file.directory:
     - name: /opt/so/rules/suricata
@@ -123,7 +124,7 @@ surirulesync:
     - name: /opt/so/rules/suricata/
     - source: salt://suricata/rules/
     - user: 940
-    - group: 940
+    - group: 939
     - show_changes: False
 
 surilogscript:
@@ -159,7 +160,6 @@ surithresholding:
     - source: salt://suricata/files/threshold.conf
     - user: 940
     - group: 940
-    - onlyif: salt://suricata/files/threshold.conf
 
 suriclassifications:
   file.managed:
@@ -177,6 +177,14 @@ so-suricata-eve-clean:
     - template: jinja
     - source: salt://suricata/cron/so-suricata-eve-clean
 
+so-suricata-rulestats:
+  file.managed:
+    - name: /usr/sbin/so-suricata-rulestats
+    - user: root
+    - group: root
+    - mode: 755
+    - source: salt://suricata/cron/so-suricata-rulestats
+
 {% else %}
 
 {{sls}}_state_not_allowed:

salt/suricata/cron/so-suricata-rulestats

@@ -0,0 +1,30 @@
+#!/bin/bash
+#
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+# Query Suricata for ruleset stats and reload time, write to JSON file for Telegraf to consume
+
+OUTFILE="/opt/so/log/suricata/rulestats.json"
+SURICATASC="docker exec so-suricata /opt/suricata/bin/suricatasc"
+SOCKET="/var/run/suricata/suricata-command.socket"
+
+query() {
+    timeout 10 $SURICATASC -c "$1" "$SOCKET" 2>/dev/null
+}
+
+STATS=$(query "ruleset-stats")
+RELOAD=$(query "ruleset-reload-time")
+
+if echo "$STATS" | jq -e '.return == "OK"' > /dev/null 2>&1; then
+    LOADED=$(echo "$STATS" | jq -r '.message[0].rules_loaded')
+    FAILED=$(echo "$STATS" | jq -r '.message[0].rules_failed')
+    LAST_RELOAD=$(echo "$RELOAD" | jq -r '.message[0].last_reload')
+
+    jq -n --argjson loaded "$LOADED" --argjson failed "$FAILED" --arg reload "$LAST_RELOAD" \
+        '{rules_loaded: $loaded, rules_failed: $failed, last_reload: $reload, return: "OK"}' > "$OUTFILE"
+else
+    echo '{"return":"FAIL"}' > "$OUTFILE"
+fi

salt/suricata/disabled.sls

@@ -23,6 +23,11 @@ clean_suricata_eve_files:
   cron.absent:
     - identifier: clean_suricata_eve_files
 
+# Remove rulestats cron
+rulestats:
+  cron.absent:
+    - identifier: suricata_rulestats
+
 {% else %}
 
 {{sls}}_state_not_allowed:

salt/suricata/enabled.sls

@@ -90,6 +90,18 @@ clean_suricata_eve_files:
     - month: '*'
     - dayweek: '*'
 
+# Add rulestats cron - runs every minute to query Suricata for rule load status
+suricata_rulestats:
+  cron.present:
+    - name: /usr/sbin/so-suricata-rulestats > /dev/null 2>&1
+    - identifier: suricata_rulestats
+    - user: root
+    - minute: '*'
+    - hour: '*'
+    - daymonth: '*'
+    - month: '*'
+    - dayweek: '*'
+
 {% else %}
 
 {{sls}}_state_not_allowed:

salt/suricata/files/threshold.conf

@@ -0,0 +1,2 @@
+# Threshold configuration generated by Security Onion
+# This file is automatically generated - do not edit manually

salt/telegraf/defaults.yaml

@@ -21,6 +21,7 @@ telegraf:
       - sostatus.sh
       - stenoloss.sh
       - suriloss.sh
+      - surirules.sh
       - zeekcaptureloss.sh
       - zeekloss.sh
     standalone:
@@ -36,6 +37,7 @@ telegraf:
       - sostatus.sh
       - stenoloss.sh
       - suriloss.sh
+      - surirules.sh
       - zeekcaptureloss.sh
       - zeekloss.sh
       - features.sh
@@ -81,6 +83,7 @@ telegraf:
       - sostatus.sh
       - stenoloss.sh
       - suriloss.sh
+      - surirules.sh
       - zeekcaptureloss.sh
       - zeekloss.sh
       - features.sh
@@ -95,6 +98,7 @@ telegraf:
       - sostatus.sh
       - stenoloss.sh
       - suriloss.sh
+      - surirules.sh
       - zeekcaptureloss.sh
       - zeekloss.sh
     idh:

salt/telegraf/scripts/surirules.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+#
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+# Read Suricata ruleset stats from JSON file written by so-suricata-rulestats cron job
+# JSON format: {"rules_loaded":45879,"rules_failed":1,"last_reload":"2025-12-04T14:10:57+0000","return":"OK"}
+# or on failure: {"return":"FAIL"}
+
+# if this script isn't already running
+if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then
+
+    STATSFILE="/var/log/suricata/rulestats.json"
+
+    # Check file exists, is less than 90 seconds old, and has valid data
+    if [ -f "$STATSFILE" ] && [ $(($(date +%s) - $(stat -c %Y "$STATSFILE"))) -lt 90 ] && jq -e '.return == "OK" and .rules_loaded != null and .rules_failed != null' "$STATSFILE" > /dev/null 2>&1; then
+        LOADED=$(jq -r '.rules_loaded' "$STATSFILE")
+        FAILED=$(jq -r '.rules_failed' "$STATSFILE")
+        RELOAD_TIME=$(jq -r '.last_reload // ""' "$STATSFILE")
+
+        echo "surirules loaded=${LOADED}i,failed=${FAILED}i,reload_time=\"${RELOAD_TIME}\",status=\"ok\""
+    else
+        echo "surirules loaded=0i,failed=0i,reload_time=\"\",status=\"unknown\""
+    fi
+
+fi
+
+exit 0

salt/zeek/files/config.zeek.ja4

@@ -11,6 +11,8 @@ export {
   option JA4S_enabled:   bool = F;
   option JA4S_raw:   bool = F;
 
+  option JA4D_enabled:  bool = F;
+
   option JA4H_enabled:   bool = F;
   option JA4H_raw:   bool = F;
 

setup/so-functions

@@ -656,11 +656,11 @@ check_requirements() {
 	fi
 	
 	if [[ $total_mem_hr -lt $req_mem ]]; then
-		whiptail_requirements_error "memory" "${total_mem_hr} GB" "${req_mem} GB"
 		if [[ $is_standalone || $is_heavynode ]]; then
 			echo "This install type will fail with less than $req_mem GB of memory. Exiting setup."
 			exit 0
 		fi
+		whiptail_requirements_error "memory" "${total_mem_hr} GB" "${req_mem} GB"
 	fi
 	if [[ $is_standalone || $is_heavynode ]]; then
 		if [[ $total_mem_hr -gt 15 && $total_mem_hr -lt 24 ]]; then
@@ -1598,16 +1598,21 @@ proxy_validate() {
 
 reserve_group_ids() {
 	# This is a hack to fix OS from taking group IDs that we need
+	logCmd "groupadd -g 920 docker"
 	logCmd "groupadd -g 928 kratos"
 	logCmd "groupadd -g 930 elasticsearch"
 	logCmd "groupadd -g 931 logstash"
 	logCmd "groupadd -g 932 kibana"
 	logCmd "groupadd -g 933 elastalert"
 	logCmd "groupadd -g 937 zeek"
+	logCmd "groupadd -g 938 salt"
+	logCmd "groupadd -g 939 socore"
 	logCmd "groupadd -g 940 suricata"
+	logCmd "groupadd -g 948 elastic-agent-pr"
+	logCmd "groupadd -g 949 elastic-agent"
 	logCmd "groupadd -g 941 stenographer"
-	logCmd "groupadd -g 945 ossec"
+	logCmd "groupadd -g 947 elastic-fleet"
-	logCmd "groupadd -g 946 cyberchef"
+	logCmd "groupadd -g 960 kafka"
 }
 
 reserve_ports() {

setup/so-setup

@@ -682,6 +682,8 @@ if ! [[ -f $install_opt_file ]]; then
 		fi
 		info "Reserving ports"
 		reserve_ports
+		info "Reserving group ids"
+		reserve_group_ids
 		info "Setting Paths"
 		# Set the paths
 		set_path
@@ -840,7 +842,10 @@ if ! [[ -f $install_opt_file ]]; then
 		if [[ $monints ]]; then
 			configure_network_sensor
 		fi
+		info "Reserving ports"
 		reserve_ports
+		info "Reserving group ids"
+		reserve_group_ids
 		# Set the version
 		mark_version
 		# Disable the setup from prompting at login