Changes from feedback

update so-stop | so-start | so-restart scripts

salt/common/tools/sbin/so-common

@@ -291,6 +291,20 @@ download_and_verify() {
 	fi
 }
 
+# check if container with name is running and optionally stop it
+docker_check_running() {
+    # show running containers, only names
+    if docker ps --format '{{.Names}}' | grep -q "^so-${1}$"; then
+        if [[ "$2" == "--stop" ]]; then
+            docker stop "so-${1}"
+        fi
+
+        return 0
+    else
+        return 1
+    fi
+}
+
 elastic_license() {
 
 read -r -d '' message <<- EOM

salt/common/tools/sbin/so-restart

@@ -5,27 +5,41 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
-
-
-# Usage: so-restart  kibana | playbook 
-
 . /usr/sbin/so-common
 
-if [ $# -ge 1 ]; then
+usage() {
+    echo "Usage: $0 <component> [args]"
+    echo ""
+    echo "Supported args:"
+    echo "  --force | -f          Force stop all Salt jobs before starting component."
+    echo ""
+    echo "Examples:"
+    echo "  $0 kibana             Restart Kibana"
+    echo "  $0 kibana --force     Force stop all Salt jobs before restarting Kibana"
+    exit 1
+}
 
-        echo $banner
-        printf "Restarting $1...\n\nThis could take a while if another Salt job is running. \nRun this command with --force to stop all Salt jobs before proceeding.\n"
-        echo $banner
-
-        if [ "$2" = "--force" ]; then
-                printf "\nForce-stopping all Salt jobs before proceeding\n\n"
-                salt-call saltutil.kill_all_jobs
-        fi
-
-        case $1 in
-                "elastic-fleet") docker stop so-elastic-fleet && docker rm so-elastic-fleet && salt-call state.apply elasticfleet queue=True;;
-                *)  docker stop so-$1 ; docker rm so-$1 ; salt-call state.apply $1 queue=True;;
-        esac
-else
-        echo -e "\nPlease provide an argument by running like so-restart $component, or by using the component-specific script.\nEx. so-restart logstash, or so-logstash-restart\n"
+if [[ $# -lt 1 ]]; then
+    usage
 fi
+
+#shellcheck disable=SC2154
+echo "$banner"
+printf "Restarting %s...\n\nThis could take a while if another Salt job is running. \nRun this command with --force to stop all Salt jobs before proceeding.\n" "$1"
+echo "$banner"
+if [[ "$2" = "--force" ]] || [[ "$2" = "-f" ]]; then
+    printf "\nForce-stopping all Salt jobs before proceeding\n\n"
+    salt-call saltutil.kill_all_jobs
+fi
+case $1 in
+    "elastic-fleet"|"elasticfleet")
+        docker_check_running "elastic-fleet" "--stop"
+        docker rm "so-elastic-fleet" 2> /dev/null
+        salt-call state.apply elasticfleet queue=True
+        ;;
+    *)
+        docker_check_running "$1" "--stop"
+        docker rm "so-${1}" 2> /dev/null
+        salt-call state.apply "$1" queue=True
+        ;;
+esac

salt/common/tools/sbin/so-start

@@ -5,27 +5,54 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
-
-
-# Usage: so-start  all | kibana | playbook 
-
+# shellcheck disable=SC1091
 . /usr/sbin/so-common
 
-if [ $# -ge 1 ]; then
-	echo $banner
-	printf "Starting $1...\n\nThis could take a while if another Salt job is running. \nRun this command with --force to stop all Salt jobs before proceeding.\n"
-	echo $banner
+usage() {
+    echo "Usage: $0 <component> [args]"
+    echo ""
+    echo "Supported args:"
+    echo "  --force | -f          Force stop all Salt jobs before starting component."
+    echo ""
+    echo "Examples:"
+    echo "  $0 kibana             Start Kibana"
+    echo "  $0 kibana --force     Force stop all Salt jobs before starting Kibana"
+    exit 1
+}
 
-	if [ "$2" = "--force" ]; then
-   		printf "\nForce-stopping all Salt jobs before proceeding\n\n"
-   		salt-call saltutil.kill_all_jobs
-	fi
-
-	case $1 in
-   		"all") salt-call state.highstate queue=True;;
-   		"elastic-fleet") if docker ps | grep -q so-$1; then printf "\n$1 is already running!\n\n"; else docker rm so-$1 >/dev/null 2>&1 ; salt-call state.apply elasticfleet queue=True; fi ;; 
-   		*) if docker ps | grep -E -q '^so-$1$'; then printf "\n$1 is already running\n\n"; else docker rm so-$1 >/dev/null 2>&1 ; salt-call state.apply $1 queue=True; fi ;; 
-	esac
-else
-	echo -e "\nPlease provide an argument by running like so-start $component, or by using the component-specific script.\nEx. so-start logstash, or so-logstash-start\n"	
+if [[ $# -lt 1 ]]; then
+    usage
 fi
+
+#shellcheck disable=SC2154
+echo "$banner"
+printf "Starting %s...\n\nThis could take a while if another Salt job is running. \nRun this command with --force to stop all Salt jobs before proceeding.\n" "$1"
+echo "$banner"
+if [[ "$2" = "--force" ]] || [[ "$2" == "-f" ]]; then
+	printf "\nForce-stopping all Salt jobs before proceeding\n\n"
+	salt-call saltutil.kill_all_jobs
+fi
+
+case "$1" in
+    "all")
+        salt-call state.highstate queue=True
+        ;;
+   	"elastic-fleet"|"elasticfleet")
+        if docker_check_running "elastic-fleet"; then
+            printf "\nso-%s is already running!\n\n" "elastic-fleet"
+            /usr/sbin/so-status
+        else
+            docker rm "so-elastic-fleet" 2> /dev/null
+            salt-call state.apply elasticfleet queue=True
+        fi
+        ;;
+   	*)
+        if docker_check_running "$1"; then
+            printf "\nso-%s is already running\n\n" "$1"
+            /usr/sbin/so-status
+        else
+            docker rm "so-${1}" 2> /dev/null
+            salt-call state.apply "$1" queue=True
+        fi
+        ;;
+esac

salt/common/tools/sbin/so-stop

@@ -5,21 +5,33 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
-
-
-# Usage: so-stop kibana | playbook | thehive
-
+# shellcheck disable=SC1091
 . /usr/sbin/so-common
 
-if [ $# -ge 1 ]; then
-	echo $banner
-	printf "Stopping $1...\n"
-	echo $banner
+usage() {
+    echo "Usage: $0 <component>"
+    echo ""
+    echo "Examples:"
+    echo "  $0 kibana             Stop Kibana"
+    exit 1
+}
 
-	case $1 in
-    		*) docker stop so-$1 ; docker rm so-$1 ;;
-	esac
-else
-	echo -e "\nPlease provide an argument by running like so-stop $component, or by using the component-specific script.\nEx. so-stop logstash, or so-logstash-stop\n"	
+if [[ $# -lt 1 ]]; then
+    usage
 fi
 
+
+#shellcheck disable=SC2154
+echo "$banner"
+printf "Stopping %s...\n" "$1"
+echo "$banner"
+case $1 in
+	"elasticfleet"|"elastic-fleet")
+        docker_check_running "elastic-fleet" "--stop"
+        docker rm "so-elastic-fleet" 2> /dev/null
+        ;;
+    *)
+        docker_check_running "$1" "--stop"
+        docker rm "so-${1}" 2> /dev/null
+    ;;
+esac

salt/common/tools/sbin_jinja/so-import-pcap

@@ -63,7 +63,8 @@ function status {
 function pcapinfo() {
   PCAP=$1
   ARGS=$2
-  docker run --rm -v "$PCAP:/input.pcap" --entrypoint capinfos {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-pcaptools:{{ VERSION }} /input.pcap -ae $ARGS
+  docker run --rm -v "$PCAP:/input.pcap" --entrypoint capinfos {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-pcaptools:{{ VERSION }} /input.pcap -ae $ARGS |\
+    sed 's/First packet/Earliest packet/g' | sed 's/Last packet/Latest packet/g'
 }
 
 function pcapfix() {

salt/elasticfleet/config.sls

@@ -173,7 +173,7 @@ eaoptionalintegrationsdir:
 
 {% for minion in node_data %}
 {% set role = node_data[minion]["role"] %}
-{% if role in [ "eval","fleet","heavynode","import","manager", "managerhype", "managersearch","standalone" ] %}
+{% if role in [ "eval","fleet","import","manager", "managerhype", "managersearch","standalone" ] %}
 {% set optional_integrations = ELASTICFLEETMERGED.optional_integrations %}
 {% set integration_keys = optional_integrations.keys() %}
 fleet_server_integrations_{{ minion }}:

salt/elasticfleet/manager.sls

@@ -67,8 +67,6 @@ so-elastic-fleet-package-upgrade:
         interval: 30
     - require:
       - http: wait_for_so-kibana
-    - onchanges:
-      - file: /opt/so/state/elastic_fleet_packages.txt
 
 so-elastic-fleet-integrations:
   cmd.run:

salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load

@@ -9,13 +9,11 @@
 RETURN_CODE=0
 
 if [ ! -f /opt/so/state/eaintegrations.txt ]; then
-  # First, check for any package upgrades
-  /usr/sbin/so-elastic-fleet-package-upgrade
 
-  # Second, update Fleet Server policies
+  # update Fleet Server policies
   /usr/sbin/so-elastic-fleet-integration-policy-elastic-fleet-server
 
-  # Third, configure Elastic Defend Integration seperately
+  # configure Elastic Defend Integration separately
   /usr/sbin/so-elastic-fleet-integration-policy-elastic-defend
 
   # Each group fetches its agent policy once and dispatches create/update writes concurrently.
@@ -32,9 +30,12 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then
   elastic_fleet_load_integrations_dir "so-grid-nodes_heavy" \
     /opt/so/conf/elastic-fleet/integrations/grid-nodes_heavy "Grid Nodes Policy_Heavy" || RETURN_CODE=1
 
-  # Fleet Server - Optional integrations (one agent policy per FleetServer_* directory)
+  # Fleet Server - Optional integrations (adds integration configuration to a given FleetServer_ policy)
   for FLEET_DIR in /opt/so/conf/elastic-fleet/integrations-optional/FleetServer*/; do
     [ -d "$FLEET_DIR" ] || continue
+    INTEGRATIONS=("${FLEET_DIR%/}"/*.json)
+    [ -e "${INTEGRATIONS[0]}" ] || continue
+
     FLEET_POLICY=$(basename "$FLEET_DIR")
     elastic_fleet_load_integrations_dir "$FLEET_POLICY" \
       "${FLEET_DIR%/}" "Fleet Server Policy" "elasticsearch-logs" || RETURN_CODE=1

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade

@@ -12,17 +12,22 @@ PKG_LOAD_FAILURES=0
 PKG_LOAD_FAILURES_NAMES=()
 
 {%- for PACKAGE in SUPPORTED_PACKAGES %}
-echo "Upgrading {{ PACKAGE }} package..."
-if VERSION=$(elastic_fleet_package_latest_version_check "{{ PACKAGE }}"); then
-    if ! elastic_fleet_package_install "{{ PACKAGE }}" "$VERSION"; then
-        PKG_LOAD_FAILURES=$((PKG_LOAD_FAILURES + 1))
-        PKG_LOAD_FAILURES_NAMES+=("{{ PACKAGE }}")
+if INSTALLED_VERSION=$(elastic_fleet_package_version_check "{{ PACKAGE }}") && LATEST_VERSION=$(elastic_fleet_package_latest_version_check "{{ PACKAGE }}"); then
+
+    if [ "$INSTALLED_VERSION" == "$LATEST_VERSION" ]; then
+        echo "{{ PACKAGE }} integration version $INSTALLED_VERSION is already at the reported latest version $LATEST_VERSION, skipping upgrade."
+    else
+        echo "Upgrading {{ PACKAGE }} package to version $LATEST_VERSION..."
+        if ! elastic_fleet_package_install "{{ PACKAGE }}" "$LATEST_VERSION"; then
+            PKG_LOAD_FAILURES=$((PKG_LOAD_FAILURES + 1))
+            PKG_LOAD_FAILURES_NAMES+=("{{ PACKAGE }}")
+        fi
     fi
 else
+    echo "ERROR: Failed to get version information for integration {{ PACKAGE }}"
     PKG_LOAD_FAILURES=$((PKG_LOAD_FAILURES + 1))
     PKG_LOAD_FAILURES_NAMES+=("{{ PACKAGE }}")
 fi
-echo
 {%- endfor %}
 
 if [ $PKG_LOAD_FAILURES -gt 0 ]; then
@@ -35,6 +40,3 @@ if [ $PKG_LOAD_FAILURES -gt 0 ]; then
 else
     echo "Successfully upgraded all packages."
 fi
-
-echo
-/usr/sbin/so-elasticsearch-templates-load

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup

@@ -181,6 +181,9 @@ if ! elastic_fleet_policy_create "so-grid-nodes_heavy" "SO Grid Nodes - Heavy No
     exit 1
 fi
 
+# Check for package upgrades
+so-elastic-fleet-package-upgrade
+
 # Load Integrations for default policies
 so-elastic-fleet-integration-policy-load
 

salt/elasticsearch/files/ingest/zeek.ssl

@@ -5,6 +5,7 @@
     { "remove":   { "field": ["host"],     "ignore_failure": true } },
     { "json":             { "field": "message",                 "target_field": "message2",             "ignore_failure": true  } },
     { "rename":         { "field": "message2.version",          "target_field": "ssl.version",          "ignore_missing": true  } },
+    { "set":      { "description": "Set transport for the community_id processor", "if": "ctx.ssl?.version == null || !ctx.ssl.version.startsWith('DTLS')", "field": "network.transport", "value": "tcp", "ignore_failure": true } },
     { "rename":         { "field": "message2.cipher",           "target_field": "ssl.cipher",           "ignore_missing": true  } },
     { "rename":         { "field": "message2.curve",            "target_field": "ssl.curve",            "ignore_missing": true  } },
     { "rename":         { "field": "message2.server_name",      "target_field": "ssl.server_name",              "ignore_missing": true  } },

salt/soc/config.sls

@@ -134,6 +134,30 @@ socsigmasopipeline:
     - group: 939
     - mode: 600
 
+socsigmaplaybookpipeline:
+  file.managed:
+    - name: /opt/so/conf/soc/sigma_playbook_pipeline.yaml
+    - source: salt://soc/files/soc/sigma_playbook_pipeline.yaml
+    - user: 939
+    - group: 939
+    - mode: 600
+
+socplaybookplaceholdermap:
+  file.managed:
+    - name: /opt/so/conf/soc/playbook_placeholder_map.yaml
+    - source: salt://soc/files/soc/playbook_placeholder_map.yaml
+    - user: 939
+    - group: 939
+    - mode: 600
+
+socplaybookplaceholdermapcustom:
+  file.managed:
+    - name: /opt/so/conf/soc/playbook_placeholder_map_custom.yaml
+    - source: salt://soc/files/soc/playbook_placeholder_map_custom.yaml
+    - user: 939
+    - group: 939
+    - mode: 600
+
 socbanner:
   file.managed:
     - name: /opt/so/conf/soc/banner.md

salt/soc/defaults.yaml

@@ -1502,6 +1502,9 @@ soc:
               - repo: https://github.com/Security-Onion-Solutions/securityonion-resources-playbooks
                 branch: main
                 folder: securityonion-normalized
+              - repo: https://github.com/Security-Onion-Solutions/securityonion-resources-playbooks
+                branch: published
+                folder: sigma
             airgap:
               - repo: file:///nsm/airgap-resources/playbooks/securityonion-resources-playbooks
                 branch: main

salt/soc/enabled.sls

@@ -45,7 +45,10 @@ so-soc:
       - /opt/so/conf/soc/motd.md:/opt/sensoroni/html/motd.md:ro
       - /opt/so/conf/soc/banner.md:/opt/sensoroni/html/login/banner.md:ro
       - /opt/so/conf/soc/sigma_so_pipeline.yaml:/opt/sensoroni/sigma_so_pipeline.yaml:ro
-      - /opt/so/conf/soc/sigma_final_pipeline.yaml:/opt/sensoroni/sigma_final_pipeline.yaml:rw
+      - /opt/so/conf/soc/sigma_playbook_pipeline.yaml:/opt/sensoroni/sigma_playbook_pipeline.yaml:ro
+      - /opt/so/conf/soc/sigma_final_pipeline.yaml:/opt/sensoroni/sigma_final_pipeline.yaml:ro
+      - /opt/so/conf/soc/playbook_placeholder_map.yaml:/opt/sensoroni/playbook_placeholder_map.yaml:ro
+      - /opt/so/conf/soc/playbook_placeholder_map_custom.yaml:/opt/sensoroni/playbook_placeholder_map_custom.yaml:ro
       - /opt/so/conf/soc/custom.js:/opt/sensoroni/html/js/custom.js:ro
       - /opt/so/conf/soc/custom_roles:/opt/sensoroni/rbac/custom_roles:ro
       - /opt/so/conf/soc/soc_users_roles:/opt/sensoroni/rbac/users_roles:rw
@@ -99,6 +102,8 @@ so-soc:
       - file: soccustomroles
       - file: socusersroles
       - file: socclientsroles
+      - file: socplaybookplaceholdermap
+      - file: socplaybookplaceholdermapcustom
 
 delete_so-soc_so-status.disabled:
   file.uncomment:

salt/soc/files/soc/playbook_placeholder_map.yaml

@@ -0,0 +1,49 @@
+# Global Playbook placeholder map: %token% -> event field path.
+#
+# Loaded by the SOC Playbook module and used to resolve `field|expand:%placeholder%` values 
+# from an alert when converting playbook questions to OQL. 
+# Left: the %token% used in a question
+# Right: the event field its value is read from (event_data.-nested or bare; the module
+# tries both). 
+#
+# Example: with `src_ip: source.ip` (below), a question that writes
+# `source.ip|expand: '%src_ip%'` resolves %src_ip% to the alert's source.ip at convert time.
+#
+# This is the global base layer. To add or override tokens edit playbook_placeholder_map_custom.yaml.
+# those entries overlay this map and win on conflict.
+
+CommandLine: process.command_line
+CurrentDirectory: process.working_directory
+Image: process.executable
+ImageLoaded: dll.name
+ParentImage: process.parent.executable
+ParentName: process.parent.name
+ParentProcessGuid: process.parent.entity_id
+ProcessGuid: process.entity_id
+TargetFilename: file.name
+TargetObject: registry.path
+TargetUserName: user.target.name
+User: user.name
+community_id: network.community_id
+dns_resolved_ip: dns.resolved_ip
+document_id: soc_id
+dst_ip: destination.ip
+dst_port: destination.port
+event_data_source_ip: source.ip
+file_path: file.path
+file_dirs: process.file_dirs
+file_name: process.name
+file_paths: process.file_paths
+hostname: host.name
+private_ip: network.private_ip
+public_ip: network.public_ip
+related_hosts: related.hosts
+related_ip: related.ip
+src_ip: source.ip
+dns_query_name: dns.query_name
+flow_id: log.id.uid
+payload: network.data.decoded
+rule_category: rule.category
+rule_name: rule.name
+rule_uuid: rule.uuid
+src_port: source.port

salt/soc/files/soc/playbook_placeholder_map_custom.yaml

@@ -0,0 +1,14 @@
+# Custom Playbook placeholder map: %token% -> event field path.
+#
+#
+# Left: the %token% used in a playbook question.
+# Right: the event field its value is read from (event_data.-nested or bare; the module tries
+# both). Note: a token that is simply named after a flat event field resolves automatically
+# without an entry here - only add a mapping when the token name differs from the field name.
+#
+# Example:
+#
+# account_id: cloudflare.account_id
+#
+# A question that writes
+# `account_id|expand: '%account_id%'` resolves %account_id% from the alert at convert time.

salt/soc/files/soc/sigma_playbook_pipeline.yaml

@@ -0,0 +1,12 @@
+name: Security Onion - Playbook Pipeline
+priority: 97
+transformations:
+    # Route string fields to their lowercase-normalized .caseless subfield so wildcard
+    # matches are case-insensitive.
+    - id: case_insensitive_string_fields
+      type: field_name_mapping
+      mapping:
+        process.executable: process.executable.caseless
+        process.parent.executable: process.parent.executable.caseless
+        process.command_line: process.command_line.caseless
+        process.parent.command_line: process.parent.command_line.caseless

salt/soc/files/soc/sigma_so_pipeline.yaml

@@ -63,6 +63,14 @@ transformations:
       rule_conditions:
       - type: logsource
         category: antivirus
+    # OS-agnostic process_creation scoping for product-less (NIDS/host-pivot) rules.
+    - id: process_creation_os_agnostic
+      type: add_condition
+      conditions:
+        event.category: process
+      rule_conditions:
+      - type: logsource
+        category: process_creation
     # Transforms the `Hashes` field to ECS fields
     # ECS fields are used by the hash fields emitted by Elastic Defend
     # If shipped with Elastic Agent, sysmon logs will also have hashes mapped to ECS fields    
@@ -108,6 +116,40 @@ transformations:
       - type: logsource
         product: windows
         category: driver_load
+    - id: ecs_fix_process_creation
+      type: field_name_mapping
+      mapping:
+        # bare `Hashes` (the combined-string case is broken out above)
+        winlog.event_data.Hashes: process.hash.sha256
+        winlog.event_data.IntegrityLevel: process.Ext.token.integrity_level_name
+        winlog.event_data.ParentName: process.parent.name
+      rule_conditions:
+      - type: logsource
+        product: windows
+        category: process_creation
+    - id: ecs_fix_registry_set
+      type: field_name_mapping
+      mapping:
+        winlog.event_data.Details: registry.data.strings
+        # field rename only; EventType values (SetValue/CreateKey) still differ from
+        # event.action values (modification/creation)
+        winlog.event_data.EventType: event.action
+      rule_conditions:
+      - type: logsource
+        product: windows
+        category: registry_set
+    - id: ecs_fix_image_load
+      type: field_name_mapping
+      mapping:
+        file.path: dll.path
+        file.code_signature.signed: dll.code_signature.exists
+        winlog.event_data.Signature: dll.code_signature.subject_name
+        file.code_signature.status: dll.code_signature.status
+        winlog.event_data.Hashes: dll.hash.sha256
+      rule_conditions:
+      - type: logsource
+        product: windows
+        category: image_load
     - id: linux_security_add-fields
       type: add_condition
       conditions:
@@ -281,6 +323,15 @@ transformations:
       rule_conditions:
       - type: logsource
         category: file_event
+    # Scope image_load rules to Elastic Endpoint library events (event.category:library, dll.*
+    # populated).
+    - id: endpoint_image_load_add-fields
+      type: add_condition
+      conditions:
+        event.category: 'library'
+      rule_conditions:
+      - type: logsource
+        category: image_load
     # Maps network rules to all network logs
     # This targets all network logs, all services, generated from endpoints and network
     - id: network_add-fields

salt/soc/soc_soc.yaml

@@ -46,7 +46,15 @@ soc:
         syntax: yaml
         file: True
         global: True
-        advanced: True
+        advanced: False
+        helpLink: security-onion-console-customization
+      playbook_placeholder_map_custom__yaml:
+        title: Playbook Placeholder Map
+        description: Custom mappings of Playbook %placeholder% tokens to event fields.
+        syntax: yaml
+        file: True
+        global: True
+        advanced: False
         helpLink: security-onion-console-customization
   config:
     licenseKey:

setup/so-setup

@@ -9,14 +9,17 @@
 # Make sure you are root before doing anything
 uid="$(id -u)"
 if [ "$uid" -ne 0 ]; then
-	echo "This script must be run using sudo!"
-	fail_setup
+	echo "This script must be run using sudo!" >&2
+	exit 1
 fi
 
 # Save the original argument array since we modify it
 original_args=("$@")
 
-cd "$(dirname "$0")" || fail_setup
+cd "$(dirname "$0")" || {
+	echo "Unable to change to setup directory" >&2
+	exit 1
+}
 
 echo "Getting started..."