Merge pull request #16039 from Security-Onion-Solutions/TOoSmOotH-patch-1

Add cluster_health option to telegraf.conf

pillar/data/addtotab.sh

@@ -1,59 +0,0 @@
-#!/usr/bin/env bash
-
-# This script adds sensors/nodes/etc to the nodes tab
-default_salt_dir=/opt/so/saltstack/default
-local_salt_dir=/opt/so/saltstack/local
-TYPE=$1
-NAME=$2
-IPADDRESS=$3
-CPUS=$4
-GUID=$5
-MANINT=$6
-ROOTFS=$7
-NSM=$8
-MONINT=$9
-#NODETYPE=$10
-#HOTNAME=$11
-
-echo "Seeing if this host is already in here. If so delete it"
-if grep -q $NAME "$local_salt_dir/pillar/data/$TYPE.sls"; then
-  echo "Node Already Present - Let's re-add it"
-  awk -v blah="  $NAME:" 'BEGIN{ print_flag=1 }
-{
-    if( $0 ~ blah )
-    {
-       print_flag=0;
-       next
-    }
-    if( $0 ~ /^  [a-zA-Z0-9]+:$/ )
-    {
-        print_flag=1;
-    }
-    if ( print_flag == 1 )
-        print $0
-
-} ' $local_salt_dir/pillar/data/$TYPE.sls > $local_salt_dir/pillar/data/tmp.$TYPE.sls
-mv $local_salt_dir/pillar/data/tmp.$TYPE.sls $local_salt_dir/pillar/data/$TYPE.sls
-echo "Deleted $NAME from the tab. Now adding it in again with updated info"
-fi
-echo "  $NAME:" >> $local_salt_dir/pillar/data/$TYPE.sls
-echo "    ip: $IPADDRESS" >> $local_salt_dir/pillar/data/$TYPE.sls
-echo "    manint: $MANINT" >> $local_salt_dir/pillar/data/$TYPE.sls
-echo "    totalcpus: $CPUS" >> $local_salt_dir/pillar/data/$TYPE.sls
-echo "    guid: $GUID" >> $local_salt_dir/pillar/data/$TYPE.sls
-echo "    rootfs: $ROOTFS" >> $local_salt_dir/pillar/data/$TYPE.sls
-echo "    nsmfs: $NSM" >> $local_salt_dir/pillar/data/$TYPE.sls
-if [ $TYPE == 'sensorstab' ]; then
-  echo "    monint: bond0" >> $local_salt_dir/pillar/data/$TYPE.sls
-fi
-if [ $TYPE == 'evaltab' ] || [ $TYPE == 'standalonetab' ]; then
-  echo "    monint: bond0" >> $local_salt_dir/pillar/data/$TYPE.sls
-  if [ ! $10 ]; then
-    salt-call state.apply utility queue=True
-  fi
-fi
-if [ $TYPE == 'nodestab' ]; then
-  salt-call state.apply elasticsearch queue=True
-#  echo "    nodetype: $NODETYPE" >> $local_salt_dir/pillar/data/$TYPE.sls
-#  echo "    hotname: $HOTNAME" >> $local_salt_dir/pillar/data/$TYPE.sls
-fi

salt/allowed_states.map.jinja

@@ -37,8 +37,7 @@
     'elasticfleet',
     'elasticfleet.manager',
     'elasticsearch.cluster',
-    'elastic-fleet-package-registry',
+    'elastic-fleet-package-registry'
-    'utility'
 ] %}
 
 {% set sensor_states = [

salt/common/tools/sbin/so-common

@@ -291,6 +291,20 @@ download_and_verify() {
 	fi
 }
 
+# check if container with name is running and optionally stop it
+docker_check_running() {
+    # show running containers, only names
+    if docker ps --format '{{.Names}}' | grep -q "^so-${1}$"; then
+        if [[ "$2" == "--stop" ]]; then
+            docker stop "so-${1}"
+        fi
+
+        return 0
+    else
+        return 1
+    fi
+}
+
 elastic_license() {
 
 read -r -d '' message <<- EOM

salt/common/tools/sbin/so-restart

@@ -5,27 +5,41 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
-
-
-# Usage: so-restart  kibana | playbook 
-
 . /usr/sbin/so-common
 
-if [ $# -ge 1 ]; then
+usage() {
+    echo "Usage: $0 <component> [args]"
+    echo ""
+    echo "Supported args:"
+    echo "  --force | -f          Force stop all Salt jobs before starting component."
+    echo ""
+    echo "Examples:"
+    echo "  $0 kibana             Restart Kibana"
+    echo "  $0 kibana --force     Force stop all Salt jobs before restarting Kibana"
+    exit 1
+}
 
-        echo $banner
+if [[ $# -lt 1 ]]; then
-        printf "Restarting $1...\n\nThis could take a while if another Salt job is running. \nRun this command with --force to stop all Salt jobs before proceeding.\n"
+    usage
-        echo $banner
-
-        if [ "$2" = "--force" ]; then
-                printf "\nForce-stopping all Salt jobs before proceeding\n\n"
-                salt-call saltutil.kill_all_jobs
-        fi
-
-        case $1 in
-                "elastic-fleet") docker stop so-elastic-fleet && docker rm so-elastic-fleet && salt-call state.apply elasticfleet queue=True;;
-                *)  docker stop so-$1 ; docker rm so-$1 ; salt-call state.apply $1 queue=True;;
-        esac
-else
-        echo -e "\nPlease provide an argument by running like so-restart $component, or by using the component-specific script.\nEx. so-restart logstash, or so-logstash-restart\n"
 fi
+
+#shellcheck disable=SC2154
+echo "$banner"
+printf "Restarting %s...\n\nThis could take a while if another Salt job is running. \nRun this command with --force to stop all Salt jobs before proceeding.\n" "$1"
+echo "$banner"
+if [[ "$2" = "--force" ]] || [[ "$2" = "-f" ]]; then
+    printf "\nForce-stopping all Salt jobs before proceeding\n\n"
+    salt-call saltutil.kill_all_jobs
+fi
+case $1 in
+    "elastic-fleet"|"elasticfleet")
+        docker_check_running "elastic-fleet" "--stop"
+        docker rm "so-elastic-fleet" 2> /dev/null
+        salt-call state.apply elasticfleet queue=True
+        ;;
+    *)
+        docker_check_running "$1" "--stop"
+        docker rm "so-${1}" 2> /dev/null
+        salt-call state.apply "$1" queue=True
+        ;;
+esac

salt/common/tools/sbin/so-start

@@ -5,27 +5,54 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
-
+# shellcheck disable=SC1091
-
-# Usage: so-start  all | kibana | playbook 
-
 . /usr/sbin/so-common
 
-if [ $# -ge 1 ]; then
+usage() {
-	echo $banner
+    echo "Usage: $0 <component> [args]"
-	printf "Starting $1...\n\nThis could take a while if another Salt job is running. \nRun this command with --force to stop all Salt jobs before proceeding.\n"
+    echo ""
-	echo $banner
+    echo "Supported args:"
+    echo "  --force | -f          Force stop all Salt jobs before starting component."
+    echo ""
+    echo "Examples:"
+    echo "  $0 kibana             Start Kibana"
+    echo "  $0 kibana --force     Force stop all Salt jobs before starting Kibana"
+    exit 1
+}
 
-	if [ "$2" = "--force" ]; then
+if [[ $# -lt 1 ]]; then
-   		printf "\nForce-stopping all Salt jobs before proceeding\n\n"
+    usage
-   		salt-call saltutil.kill_all_jobs
-	fi
-
-	case $1 in
-   		"all") salt-call state.highstate queue=True;;
-   		"elastic-fleet") if docker ps | grep -q so-$1; then printf "\n$1 is already running!\n\n"; else docker rm so-$1 >/dev/null 2>&1 ; salt-call state.apply elasticfleet queue=True; fi ;; 
-   		*) if docker ps | grep -E -q '^so-$1$'; then printf "\n$1 is already running\n\n"; else docker rm so-$1 >/dev/null 2>&1 ; salt-call state.apply $1 queue=True; fi ;; 
-	esac
-else
-	echo -e "\nPlease provide an argument by running like so-start $component, or by using the component-specific script.\nEx. so-start logstash, or so-logstash-start\n"	
 fi
+
+#shellcheck disable=SC2154
+echo "$banner"
+printf "Starting %s...\n\nThis could take a while if another Salt job is running. \nRun this command with --force to stop all Salt jobs before proceeding.\n" "$1"
+echo "$banner"
+if [[ "$2" = "--force" ]] || [[ "$2" == "-f" ]]; then
+	printf "\nForce-stopping all Salt jobs before proceeding\n\n"
+	salt-call saltutil.kill_all_jobs
+fi
+
+case "$1" in
+    "all")
+        salt-call state.highstate queue=True
+        ;;
+   	"elastic-fleet"|"elasticfleet")
+        if docker_check_running "elastic-fleet"; then
+            printf "\nso-%s is already running!\n\n" "elastic-fleet"
+            /usr/sbin/so-status
+        else
+            docker rm "so-elastic-fleet" 2> /dev/null
+            salt-call state.apply elasticfleet queue=True
+        fi
+        ;;
+   	*)
+        if docker_check_running "$1"; then
+            printf "\nso-%s is already running\n\n" "$1"
+            /usr/sbin/so-status
+        else
+            docker rm "so-${1}" 2> /dev/null
+            salt-call state.apply "$1" queue=True
+        fi
+        ;;
+esac

salt/common/tools/sbin/so-stop

@@ -5,21 +5,33 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
-
+# shellcheck disable=SC1091
-
-# Usage: so-stop kibana | playbook | thehive
-
 . /usr/sbin/so-common
 
-if [ $# -ge 1 ]; then
+usage() {
-	echo $banner
+    echo "Usage: $0 <component>"
-	printf "Stopping $1...\n"
+    echo ""
-	echo $banner
+    echo "Examples:"
+    echo "  $0 kibana             Stop Kibana"
+    exit 1
+}
 
-	case $1 in
+if [[ $# -lt 1 ]]; then
-    		*) docker stop so-$1 ; docker rm so-$1 ;;
+    usage
-	esac
-else
-	echo -e "\nPlease provide an argument by running like so-stop $component, or by using the component-specific script.\nEx. so-stop logstash, or so-logstash-stop\n"	
 fi
 
+
+#shellcheck disable=SC2154
+echo "$banner"
+printf "Stopping %s...\n" "$1"
+echo "$banner"
+case $1 in
+	"elasticfleet"|"elastic-fleet")
+        docker_check_running "elastic-fleet" "--stop"
+        docker rm "so-elastic-fleet" 2> /dev/null
+        ;;
+    *)
+        docker_check_running "$1" "--stop"
+        docker rm "so-${1}" 2> /dev/null
+    ;;
+esac

salt/kibana/enabled.sls

@@ -69,7 +69,7 @@ wait_for_so-kibana:
     - ssl: True
     - verify_ssl: False
     - status: 200
-    - wait_for: 300
+    - wait_for: 600
     - request_interval: 15
     - require:
       - docker_container: so-kibana

salt/manager/files/repodownload.conf

@@ -11,8 +11,8 @@ name=Security Onion Repo repo
 mirrorlist=file:///opt/so/conf/reposync/mirror.txt
 enabled=1
 gpgcheck=1
-[securityonionkernel]
+[securityonionkernelsync]
-name=Security Onion Repo repo
+name=Security Onion Kernel Repo repo
 mirrorlist=file:///opt/so/conf/reposync/mirror-kernel.txt
 enabled=1
 gpgcheck=1

salt/manager/tools/sbin/so-repo-sync

@@ -17,9 +17,9 @@ createrepo /nsm/repo
 # The kernel repo section is deployed to repodownload.conf by the manager highstate, which
 # runs AFTER this script during soup. On the first upgrade to a kernel-aware version the
 # on-disk config still predates the section, so guard on its presence to avoid dnf's
-# "Unknown repo: 'securityonionkernel'" aborting the sync (set -e). The next sync after the
+# "Unknown repo: 'securityonionkernelsync'" aborting the sync (set -e). The next sync after the
 # highstate deploys the section will pick it up.
-if grep -q '^\[securityonionkernel\]' /opt/so/conf/reposync/repodownload.conf; then
+if grep -q '^\[securityonionkernelsync\]' /opt/so/conf/reposync/repodownload.conf; then
-  dnf reposync --norepopath -g --delete -m -c /opt/so/conf/reposync/repodownload.conf --repoid=securityonionkernel --download-metadata -p /nsm/kernelrepo/
+  dnf reposync --norepopath -g --delete -m -c /opt/so/conf/reposync/repodownload.conf --repoid=securityonionkernelsync --download-metadata -p /nsm/kernelrepo/
   createrepo /nsm/kernelrepo
 fi

salt/manager/tools/sbin/soup

@@ -245,6 +245,7 @@ check_airgap() {
       UPDATE_DIR=/tmp/soagupdate/SecurityOnion
       AGDOCKER=/tmp/soagupdate/docker
       AGREPO=/tmp/soagupdate/minimal/Packages
+      AGUEKREPO=/tmp/soagupdate/uek/Packages
   else
       is_airgap=1
   fi
@@ -1004,13 +1005,19 @@ update_airgap_rules() {
   rsync -a $UPDATE_DIR/agrules/securityonion-resources/* /nsm/securityonion-resources/
 }
 
-update_airgap_repo() {
+update_airgap_repos() {
   # Update the files in the repo
-  echo "Syncing new updates to /nsm/repo"
+  echo "Syncing new updates to /nsm/repo & /nsm/kernelrepo"
-  rsync -a $AGREPO/* /nsm/repo/
+  # Airgap soup copies new files into the local repo, but doesn't remove old packages. Retaining the ability to rollback package updates
-  echo "Creating repo"
+  rsync -a "$AGREPO"/ /nsm/repo/
+  rsync -a "$AGUEKREPO"/ /nsm/kernelrepo/
+
   dnf -y install yum-utils createrepo_c
+
+  echo "Running createrepo for /nsm/repo"
   createrepo /nsm/repo
+  echo "Running createrepo for /nsm/kernelrepo"
+  createrepo /nsm/kernelrepo
 }
 
 update_salt_mine() {
@@ -1766,7 +1773,7 @@ main() {
   set -e
 
   if [[ $is_airgap -eq 0 ]]; then
-    update_airgap_repo
+    update_airgap_repos
     dnf clean all
     check_os_updates
   elif [[ $OS == 'oracle' ]]; then

salt/soc/config.sls

@@ -134,6 +134,30 @@ socsigmasopipeline:
     - group: 939
     - mode: 600
 
+socsigmaplaybookpipeline:
+  file.managed:
+    - name: /opt/so/conf/soc/sigma_playbook_pipeline.yaml
+    - source: salt://soc/files/soc/sigma_playbook_pipeline.yaml
+    - user: 939
+    - group: 939
+    - mode: 600
+
+socplaybookplaceholdermap:
+  file.managed:
+    - name: /opt/so/conf/soc/playbook_placeholder_map.yaml
+    - source: salt://soc/files/soc/playbook_placeholder_map.yaml
+    - user: 939
+    - group: 939
+    - mode: 600
+
+socplaybookplaceholdermapcustom:
+  file.managed:
+    - name: /opt/so/conf/soc/playbook_placeholder_map_custom.yaml
+    - source: salt://soc/files/soc/playbook_placeholder_map_custom.yaml
+    - user: 939
+    - group: 939
+    - mode: 600
+
 socbanner:
   file.managed:
     - name: /opt/so/conf/soc/banner.md

salt/soc/defaults.yaml

@@ -1502,6 +1502,9 @@ soc:
               - repo: https://github.com/Security-Onion-Solutions/securityonion-resources-playbooks
                 branch: main
                 folder: securityonion-normalized
+              - repo: https://github.com/Security-Onion-Solutions/securityonion-resources-playbooks
+                branch: published
+                folder: sigma
             airgap:
               - repo: file:///nsm/airgap-resources/playbooks/securityonion-resources-playbooks
                 branch: main
@@ -1771,13 +1774,13 @@ soc:
               enabled: true
           queries:
             - name: Default Query
-              description: Show all events grouped by the observer host
-              query: '* | groupby observer.name'
-              showSubtitle: true
-            - name: Log Type
               description: Show all events grouped by module and dataset
               query: '* | groupby event.module* event.dataset'
               showSubtitle: true
+            - name: Observer
+              description: Show all events grouped by the observer host
+              query: '* | groupby observer.name'
+              showSubtitle: true
             - name: SOC - Auth
               description: Users authenticated to SOC grouped by IP address and identity
               query: 'event.dataset:kratos.audit AND msg:*authenticated* | groupby http.request.headers.x-real-ip user.name'

salt/soc/enabled.sls

@@ -45,7 +45,10 @@ so-soc:
       - /opt/so/conf/soc/motd.md:/opt/sensoroni/html/motd.md:ro
       - /opt/so/conf/soc/banner.md:/opt/sensoroni/html/login/banner.md:ro
       - /opt/so/conf/soc/sigma_so_pipeline.yaml:/opt/sensoroni/sigma_so_pipeline.yaml:ro
-      - /opt/so/conf/soc/sigma_final_pipeline.yaml:/opt/sensoroni/sigma_final_pipeline.yaml:rw
+      - /opt/so/conf/soc/sigma_playbook_pipeline.yaml:/opt/sensoroni/sigma_playbook_pipeline.yaml:ro
+      - /opt/so/conf/soc/sigma_final_pipeline.yaml:/opt/sensoroni/sigma_final_pipeline.yaml:ro
+      - /opt/so/conf/soc/playbook_placeholder_map.yaml:/opt/sensoroni/playbook_placeholder_map.yaml:ro
+      - /opt/so/conf/soc/playbook_placeholder_map_custom.yaml:/opt/sensoroni/playbook_placeholder_map_custom.yaml:ro
       - /opt/so/conf/soc/custom.js:/opt/sensoroni/html/js/custom.js:ro
       - /opt/so/conf/soc/custom_roles:/opt/sensoroni/rbac/custom_roles:ro
       - /opt/so/conf/soc/soc_users_roles:/opt/sensoroni/rbac/users_roles:rw
@@ -99,6 +102,8 @@ so-soc:
       - file: soccustomroles
       - file: socusersroles
       - file: socclientsroles
+      - file: socplaybookplaceholdermap
+      - file: socplaybookplaceholdermapcustom
 
 delete_so-soc_so-status.disabled:
   file.uncomment:

salt/soc/files/soc/playbook_placeholder_map.yaml

@@ -0,0 +1,49 @@
+# Global Playbook placeholder map: %token% -> event field path.
+#
+# Loaded by the SOC Playbook module and used to resolve `field|expand:%placeholder%` values 
+# from an alert when converting playbook questions to OQL. 
+# Left: the %token% used in a question
+# Right: the event field its value is read from (event_data.-nested or bare; the module
+# tries both). 
+#
+# Example: with `src_ip: source.ip` (below), a question that writes
+# `source.ip|expand: '%src_ip%'` resolves %src_ip% to the alert's source.ip at convert time.
+#
+# This is the global base layer. To add or override tokens edit playbook_placeholder_map_custom.yaml.
+# those entries overlay this map and win on conflict.
+
+CommandLine: process.command_line
+CurrentDirectory: process.working_directory
+Image: process.executable
+ImageLoaded: dll.name
+ParentImage: process.parent.executable
+ParentName: process.parent.name
+ParentProcessGuid: process.parent.entity_id
+ProcessGuid: process.entity_id
+TargetFilename: file.name
+TargetObject: registry.path
+TargetUserName: user.target.name
+User: user.name
+community_id: network.community_id
+dns_resolved_ip: dns.resolved_ip
+document_id: soc_id
+dst_ip: destination.ip
+dst_port: destination.port
+event_data_source_ip: source.ip
+file_path: file.path
+file_dirs: process.file_dirs
+file_name: process.name
+file_paths: process.file_paths
+hostname: host.name
+private_ip: network.private_ip
+public_ip: network.public_ip
+related_hosts: related.hosts
+related_ip: related.ip
+src_ip: source.ip
+dns_query_name: dns.query_name
+flow_id: log.id.uid
+payload: network.data.decoded
+rule_category: rule.category
+rule_name: rule.name
+rule_uuid: rule.uuid
+src_port: source.port

salt/soc/files/soc/playbook_placeholder_map_custom.yaml

@@ -0,0 +1,14 @@
+# Custom Playbook placeholder map: %token% -> event field path.
+#
+#
+# Left: the %token% used in a playbook question.
+# Right: the event field its value is read from (event_data.-nested or bare; the module tries
+# both). Note: a token that is simply named after a flat event field resolves automatically
+# without an entry here - only add a mapping when the token name differs from the field name.
+#
+# Example:
+#
+# account_id: cloudflare.account_id
+#
+# A question that writes
+# `account_id|expand: '%account_id%'` resolves %account_id% from the alert at convert time.

salt/soc/files/soc/sigma_playbook_pipeline.yaml

@@ -0,0 +1,12 @@
+name: Security Onion - Playbook Pipeline
+priority: 97
+transformations:
+    # Route string fields to their lowercase-normalized .caseless subfield so wildcard
+    # matches are case-insensitive.
+    - id: case_insensitive_string_fields
+      type: field_name_mapping
+      mapping:
+        process.executable: process.executable.caseless
+        process.parent.executable: process.parent.executable.caseless
+        process.command_line: process.command_line.caseless
+        process.parent.command_line: process.parent.command_line.caseless

salt/soc/files/soc/sigma_so_pipeline.yaml

@@ -63,6 +63,14 @@ transformations:
       rule_conditions:
       - type: logsource
         category: antivirus
+    # OS-agnostic process_creation scoping for product-less (NIDS/host-pivot) rules.
+    - id: process_creation_os_agnostic
+      type: add_condition
+      conditions:
+        event.category: process
+      rule_conditions:
+      - type: logsource
+        category: process_creation
     # Transforms the `Hashes` field to ECS fields
     # ECS fields are used by the hash fields emitted by Elastic Defend
     # If shipped with Elastic Agent, sysmon logs will also have hashes mapped to ECS fields    
@@ -108,6 +116,40 @@ transformations:
       - type: logsource
         product: windows
         category: driver_load
+    - id: ecs_fix_process_creation
+      type: field_name_mapping
+      mapping:
+        # bare `Hashes` (the combined-string case is broken out above)
+        winlog.event_data.Hashes: process.hash.sha256
+        winlog.event_data.IntegrityLevel: process.Ext.token.integrity_level_name
+        winlog.event_data.ParentName: process.parent.name
+      rule_conditions:
+      - type: logsource
+        product: windows
+        category: process_creation
+    - id: ecs_fix_registry_set
+      type: field_name_mapping
+      mapping:
+        winlog.event_data.Details: registry.data.strings
+        # field rename only; EventType values (SetValue/CreateKey) still differ from
+        # event.action values (modification/creation)
+        winlog.event_data.EventType: event.action
+      rule_conditions:
+      - type: logsource
+        product: windows
+        category: registry_set
+    - id: ecs_fix_image_load
+      type: field_name_mapping
+      mapping:
+        file.path: dll.path
+        file.code_signature.signed: dll.code_signature.exists
+        winlog.event_data.Signature: dll.code_signature.subject_name
+        file.code_signature.status: dll.code_signature.status
+        winlog.event_data.Hashes: dll.hash.sha256
+      rule_conditions:
+      - type: logsource
+        product: windows
+        category: image_load
     - id: linux_security_add-fields
       type: add_condition
       conditions:
@@ -281,6 +323,15 @@ transformations:
       rule_conditions:
       - type: logsource
         category: file_event
+    # Scope image_load rules to Elastic Endpoint library events (event.category:library, dll.*
+    # populated).
+    - id: endpoint_image_load_add-fields
+      type: add_condition
+      conditions:
+        event.category: 'library'
+      rule_conditions:
+      - type: logsource
+        category: image_load
     # Maps network rules to all network logs
     # This targets all network logs, all services, generated from endpoints and network
     - id: network_add-fields

salt/soc/soc_soc.yaml

@@ -46,7 +46,15 @@ soc:
         syntax: yaml
         file: True
         global: True
-        advanced: True
+        advanced: False
+        helpLink: security-onion-console-customization
+      playbook_placeholder_map_custom__yaml:
+        title: Playbook Placeholder Map
+        description: Custom mappings of Playbook %placeholder% tokens to event fields.
+        syntax: yaml
+        file: True
+        global: True
+        advanced: False
         helpLink: security-onion-console-customization
   config:
     licenseKey:

salt/suricata/enabled.sls

@@ -69,6 +69,7 @@ surirulereload:
     - name: /usr/sbin/so-suricata-reload-rules >> /opt/so/log/suricata/reload.log 2>&1
     - onchanges:
         - file: surirulesync
+    - onlyif: test -f /opt/so/rules/suricata/all-rulesets.rules
     - require:
         - docker_container: so-suricata
     

salt/suricata/tools/sbin/so-suricata-reload-rules

@@ -7,5 +7,59 @@
 
 . /usr/sbin/so-common
 
-retry 60 3 'docker exec so-suricata /opt/suricata/bin/suricatasc -c reload-rules /var/run/suricata/suricata-command.socket' '{"message":"done","return":"OK"}' || fail "The Suricata container was not ready in time."
+RULES_FILE="/opt/so/rules/suricata/all-rulesets.rules"
-retry 60 3 'docker exec so-suricata /opt/suricata/bin/suricatasc -c ruleset-reload-nonblocking /var/run/suricata/suricata-command.socket' '{"message":"done","return":"OK"}' || fail "The Suricata container was not ready in time."
+SOCKET="/var/run/suricata/suricata-command.socket"
+SURICATASC="docker exec so-suricata /opt/suricata/bin/suricatasc"
+
+# Format an epoch as a human-readable local timestamp for log messages.
+fmt_time() { date -d "@$1" '+%Y-%m-%d %H:%M:%S %Z' 2>/dev/null; }
+
+# Prefix each input line with the current timestamp.
+timestamp_lines() { while IFS= read -r line; do printf '%s %s\n' "$(date '+%Y-%m-%d %H:%M:%S %Z')" "$line"; done; }
+
+# Epoch of Suricata's last *completed* ruleset reload; non-zero return on failure.
+suricata_reload_epoch() {
+  local out ts
+  out=$($SURICATASC -c ruleset-reload-time "$SOCKET" 2>/dev/null)
+  ts=$(echo "$out" | jq -r '.message[0].last_reload // empty' 2>/dev/null)
+  [ -n "$ts" ] || return 1
+  date -d "$ts" +%s 2>/dev/null
+}
+
+# Trigger a fresh reload and confirm Suricata is running a ruleset at least as new
+# as the rules file. Returns 0 only when both hold, so retry keeps going until an
+# in-progress reload clears and our own reload completes.
+reload_and_verify() {
+  local out reload_epoch
+  out=$($SURICATASC -c reload-rules "$SOCKET")
+  echo "reload-rules: $out"
+
+  if [[ "$out" =~ "Reload already in progress" ]]; then
+    echo "A reload is already in progress; waiting for it to clear so a fresh reload can load the current ruleset."
+    return 1
+  fi
+  if [[ ! "$out" =~ '{"message":"done","return":"OK"}' ]]; then
+    echo "Suricata not ready or unexpected reload output; will retry."
+    return 1
+  fi
+
+  reload_epoch=$(suricata_reload_epoch) || { echo "Could not read ruleset-reload-time; will retry."; return 1; }
+  if [ "$reload_epoch" -ge "$target_mtime" ]; then
+    echo "Loaded ruleset is current: last reload ($(fmt_time "$reload_epoch")) is newer than rules file ($(fmt_time "$target_mtime"))."
+    return 0
+  fi
+  echo "Loaded ruleset is stale: last reload ($(fmt_time "$reload_epoch")) is older than rules file ($(fmt_time "$target_mtime")); retrying."
+  return 1
+}
+
+# Run the reload/verify, timestamping every line of output (ours and the
+# retry/fail helpers') so reload.log shows when each step ran. The pipeline is
+# synchronous, so the log is fully flushed and ordered before we exit; the
+# script's real exit code is preserved via PIPESTATUS.
+{
+  # Epoch mtime of the ruleset we need Suricata to have loaded. Captured once so
+  # a file update mid-reload does not move the goalpost.
+  target_mtime=$(stat -c %Y "$RULES_FILE") || fail "Could not stat the Suricata rules file: $RULES_FILE"
+  retry 60 3 'reload_and_verify' || fail "Suricata did not load the current ruleset in time."
+} 2>&1 | timestamp_lines
+exit "${PIPESTATUS[0]}"

salt/telegraf/etc/telegraf.conf

@@ -244,6 +244,7 @@
   username = "{{ ES_USER }}"
   password = "{{ ES_PASS }}"
   insecure_skip_verify = true
+  cluster_health = true
 {%- elif grains['role'] in ['so-searchnode']  %}
 [[inputs.elasticsearch]]
   servers = ["https://{{ NODEIP }}:9200"]

salt/top.sls

@@ -83,7 +83,6 @@ base:
     - zeek
     - strelka
     - elastalert
-    - utility
     - elasticfleet
     - pcap.cleanup
 
@@ -113,7 +112,6 @@ base:
     - zeek
     - strelka
     - elastalert
-    - utility
     - elasticfleet
     - stig
     - kafka
@@ -141,7 +139,6 @@ base:
     - elastic-fleet-package-registry
     - kibana
     - elastalert
-    - utility
     - elasticfleet
     - stig
     - kafka
@@ -168,7 +165,6 @@ base:
     - elastic-fleet-package-registry
     - kibana
     - elastalert
-    - utility
     - elasticfleet
     - kafka
 
@@ -198,7 +194,6 @@ base:
     - elastic-fleet-package-registry
     - kibana
     - elastalert
-    - utility
     - elasticfleet
     - stig
     - kafka
@@ -222,7 +217,6 @@ base:
     - elasticsearch
     - elastic-fleet-package-registry
     - kibana
-    - utility
     - suricata
     - zeek
     - elasticfleet

salt/utility/bin/eval

@@ -1,29 +0,0 @@
-#!/bin/bash
-# Wait for ElasticSearch to come up, so that we can query for version infromation
-echo -n "Waiting for ElasticSearch..."
-COUNT=0
-ELASTICSEARCH_CONNECTED="no"
-while [[ "$COUNT" -le 30 ]]; do
-  curl -K /opt/so/conf/elasticsearch/curl.config -k --output /dev/null --silent --head --fail -L https://{{ GLOBALS.manager_ip }}:9200
-  if [ $? -eq 0 ]; then
-    ELASTICSEARCH_CONNECTED="yes"
-    echo "connected!"
-    break
-  else
-    ((COUNT+=1))
-    sleep 1
-    echo -n "."
-  fi
-done
-if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then
-  echo
-  echo -e "Connection attempt timed out.  Unable to connect to ElasticSearch.  \nPlease try: \n  -checking log(s) in /var/log/elasticsearch/\n  -running 'docker ps' \n  -running 'sudo so-elastic-restart'"
-  echo
-
-  exit
-fi
-
-echo "Applying cross cluster search config..."
-    curl -K /opt/so/conf/elasticsearch/curl.config -s -k -XPUT -L https://{{ GLOBALS.manager_ip }}:9200/_cluster/settings \
-         -H 'Content-Type: application/json' \
-         -d "{\"persistent\": {\"search\": {\"remote\": {\"{{ grains.host }}\": {\"seeds\": [\"127.0.0.1:9300\"]}}}}}"

salt/utility/init.sls

@@ -1,22 +0,0 @@
-{% from 'allowed_states.map.jinja' import allowed_states %}
-{% from 'vars/globals.map.jinja' import GLOBALS %}
-
-{% if sls in allowed_states %}
-  {% if grains['role'] in ['so-eval', 'so-import'] %}
-fixsearch:
-  cmd.script:
-    - shell: /bin/bash
-    - cwd: /opt/so
-    - source: salt://utility/bin/eval
-    - template: jinja
-    - defaults:
-        GLOBALS: {{ GLOBALS }}
-  {% endif %}
-
-{% else %}
-
-{{sls}}_state_not_allowed:
-  test.fail_without_changes:
-    - name: {{sls}}_state_not_allowed
-
-{% endif %}

setup/so-functions

@@ -29,8 +29,12 @@ title() {
 }
 
 fail_setup() {
+	local err_msg=$1
+	if [[ -n "$err_msg" ]]; then
+		error "$err_msg"
+	fi
 	error "Setup encountered an unrecoverable failure, exiting"
-	touch /root/failure
+	echo "setup incomplete: $err_msg" > /root/failure
 	exit 1
 }
 
@@ -697,7 +701,7 @@ compare_main_nic_ip() {
 			EOM
 
 		  [[ -n $TESTING ]] || whiptail --title "$whiptail_title" --msgbox "$message" 11 75
-		  kill -SIGINT "$(ps --pid $$ -oppid=)"; fail_setup
+		  kill -SIGINT "$(ps --pid $$ -oppid=)"; fail_setup "Main IP mismatch"
 	  fi
 	else
 		# Setup uses MAINIP, but since we ignore the equality condition when using a VPN 
@@ -755,8 +759,7 @@ configure_management_bond() {
 	info "Setting up $bond_name management interface with mode $bond_mode"
 
 	if [[ ${#MBNICS[@]} -eq 0 ]]; then
-		error "[ERROR] No management bond NICs were selected."
+		fail_setup "No management bond NICs selected"
-		fail_setup
 	fi
 
 	nmcli -t -f NAME con show | grep -Fxq "$bond_name"
@@ -914,8 +917,7 @@ detect_os() {
 		is_rpm=true
 		is_supported=true
 	else
-		info "This OS is not supported. Security Onion requires Oracle Linux 9."
+		fail_setup "This OS is not supported. Security Onion requires Oracle Linux 9."
-		fail_setup
 	fi
 
 	info "Found OS: $OS $OSVER"
@@ -923,7 +925,7 @@ detect_os() {
 
 download_elastic_agent_artifacts() {
 	if ! update_elastic_agent 2>&1 | tee -a "$setup_log"; then
-		fail_setup
+		fail_setup "Failed to update Elastic Agent"
 	fi
 }
 
@@ -1567,7 +1569,7 @@ proxy_validate() {
 		error "Received error: $proxy_test_err"
 		if [[ -n $TESTING ]]; then
 			error "Exiting setup"
-			kill -SIGINT "$(ps --pid $$ -oppid=)"; fail_setup
+			kill -SIGINT "$(ps --pid $$ -oppid=)"; fail_setup "Proxy validation failed"
 		fi
 	fi
 	return $ret
@@ -1774,8 +1776,7 @@ ensure_pyyaml() {
 	local result=$?
 	set +o pipefail
 	if [[ $result -ne 0 ]] || ! rpm -q python3-pyyaml >/dev/null 2>&1; then
-		error "Failed to install python3-pyyaml (exit=$result)"
+		fail_setup "Failed to install python3-pyyaml (exit=$result)"
-		fail_setup
 	fi
 	info "python3-pyyaml installed successfully"
 }
@@ -1910,8 +1911,8 @@ repo_sync_local() {
 
 	if [[ ! $is_airgap ]]; then
 		curl --retry 5 --retry-delay 60 -A "netinstall/$SOVERSION/$OS/$(uname -r)/1" https://sigs.securityonion.net/checkup --output /tmp/install
-		retry 5 60 "dnf reposync --norepopath -g --delete -m -c /opt/so/conf/reposync/repodownload.conf --repoid=securityonionsync --download-metadata -p /nsm/repo/" >> "$setup_log" 2>&1 || fail_setup
+		retry 5 60 "dnf reposync --norepopath -g --delete -m -c /opt/so/conf/reposync/repodownload.conf --repoid=securityonionsync --download-metadata -p /nsm/repo/" >> "$setup_log" 2>&1 || fail_setup "Failed to sync repos"
-		retry 5 60 "dnf reposync --norepopath -g --delete -m -c /opt/so/conf/reposync/repodownload.conf --repoid=securityonionkernel --download-metadata -p /nsm/kernelrepo/" >> "$setup_log" 2>&1 || fail_setup
+		retry 5 60 "dnf reposync --norepopath -g --delete -m -c /opt/so/conf/reposync/repodownload.conf --repoid=securityonionkernel --download-metadata -p /nsm/kernelrepo/" >> "$setup_log" 2>&1 || fail_setup "Failed to sync kernel repos"
 		# After the download is complete run createrepo
 		create_repo
 	fi
@@ -1924,10 +1925,10 @@ saltify() {
 
 	if [[ $waitforstate ]]; then
 		# install all for a manager
-		retry 30 10 "bash ../salt/salt/scripts/bootstrap-salt.sh -r -M -X stable $SALTVERSION" || fail_setup
+		retry 30 10 "bash ../salt/salt/scripts/bootstrap-salt.sh -r -M -X stable $SALTVERSION" || fail_setup "Failed to install salt master"
 	else
 		# just a minion
-		retry 30 10 "bash ../salt/salt/scripts/bootstrap-salt.sh -r -X stable $SALTVERSION" || fail_setup
+		retry 30 10 "bash ../salt/salt/scripts/bootstrap-salt.sh -r -X stable $SALTVERSION" || fail_setup "Failed to install salt minion"
 	fi
 
 	salt_install_module_deps
@@ -1999,7 +2000,7 @@ set_main_ip() {
 			info "MAINIP=$MAINIP"
 			info "MNIC_IP=$MNIC_IP"
 			whiptail_error_message "The management IP could not be determined. Please check the log at /root/sosetup.log and verify the network configuration. Select OK to exit."
-			fail_setup
+			fail_setup "Could not determine MAINIP or MNIC_IP"
 		fi
 		sleep 1
 	done
@@ -2203,7 +2204,7 @@ set_initial_firewall_access() {
 set_management_interface() {
 	title "Setting up the main interface"
 	if [[ $MNIC == "bond1" ]]; then
-		configure_management_bond || fail_setup
+		configure_management_bond || fail_setup "Failed to configure management bond"
 	fi
 
 	if [ "$address_type" = 'DHCP' ]; then

setup/so-setup

@@ -90,8 +90,7 @@ if [[ "$setup_type" == 'iso' ]]; then
 	if [[ $is_rpm ]]; then
 		is_iso=true
 	else
-		echo "Only use 'so-setup iso' for an ISO install on Security Onion ISO images. Please run 'so-setup network' instead."
+		fail_setup "Only use 'so-setup iso' for an ISO install on Security Onion ISO images. Please run 'so-setup network' instead."
-		fail_setup
 	fi
 fi
 
@@ -130,7 +129,7 @@ catch() {
 	info "Fatal error occurred at $1 in so-setup, failing setup."
 	grep --color=never "ERROR" "$setup_log" > "$error_log"
 	whiptail_setup_failed
-	fail_setup
+	fail_setup "Fatal error occurred at $1 in so-setup"
 }
 
 # Add the progress function for manager node type installs
@@ -238,8 +237,7 @@ case "$setup_type" in
 		info "Beginning Security Onion $setup_type install"
 		;;
 	*)
-		error "Invalid install type, must be 'iso', 'network' or 'desktop'."
+		fail_setup "Invalid install type, must be 'iso', 'network' or 'desktop'."
-		fail_setup
 		;;
 esac
 
@@ -773,8 +771,7 @@ if ! [[ -f $install_opt_file ]]; then
 		logCmd "salt-call state.apply -l info registry"
 		title "Seeding the docker registry"
 		if ! docker_seed_registry; then
-			error "Failed to seed the docker registry"
+			fail_setup "Failed to seed the docker registry"
-			fail_setup
 		fi
 		title "Applying the manager state"
 		logCmd "salt-call state.apply -l info manager"
@@ -797,8 +794,7 @@ if ! [[ -f $install_opt_file ]]; then
 		title "Setting up Elastic Fleet"
 		logCmd "salt-call state.apply elasticfleet.config"	
 		if ! logCmd so-elastic-fleet-setup; then
-			error "Failed to run so-elastic-fleet-setup"
+			fail_setup "Failed to run so-elastic-fleet-setup"
-			fail_setup
 		fi
 		mark_setup_complete
 		set_initial_firewall_access

setup/so-verify

@@ -143,15 +143,15 @@ main() {
         cat $error_log
         echo "--------------------------"
         exit_code=1
-        touch /root/failure
+        echo "Found setup errors. Check $error_log for details" > /root/failure
     elif using_iso && cron_error_in_mail_spool; then
         echo "WARNING: Unexpected cron job output in mail spool"
         exit_code=1
-        touch /root/failure
+        echo "Unexpected cron job output found in /var/spool/mail/" > /root/failure
     elif is_manager_node && status_failed; then
         echo "WARNING: Containers are not in a healthy state"
         exit_code=1
-        touch /root/failure
+        echo "Containers are not in a healthy state. Check so-status for details" > /root/failure
     else
         echo "Successfully completed setup!"
         touch /root/success