From 23f04e28665af7569a7e837f5c66db87547a0c21 Mon Sep 17 00:00:00 2001 From: Matthew Wright Date: Mon, 15 Jun 2026 11:40:01 -0400 Subject: [PATCH 01/28] maxSubSessionTokens and maxDelegationDepth config settings --- salt/soc/defaults.yaml | 2 ++ salt/soc/soc_soc.yaml | 10 ++++++++++ 2 files changed, 12 insertions(+) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 7e8e76094..afb4c37d1 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1509,6 +1509,8 @@ soc: assistant: systemPromptAddendum: "" systemPromptAddendumMaxLength: 50000 + maxSubSessionTokens: 0 + maxDelegationDepth: 0 adapters: - name: SOAI protocol: securityonion_ai_cloud diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index 19853196a..cf483c406 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -719,6 +719,16 @@ soc: description: Maximum length of the system prompt addendum. Longer prompts will be truncated. global: True advanced: True + maxSubSessionTokens: + description: Maximum number of output tokens a delegated sub-session may generate across all of its turns. When the budget is reached, the sub-agent is halted and its result is returned to the parent agent. Set to 0 to disable the limit. + global: True + advanced: True + forcedType: int + maxDelegationDepth: + description: Maximum delegation nesting depth for sub-agents. For example, a value of 2 lets the main agent delegate to a sub-agent that may itself delegate one level deeper. Any deeper delegation is refused and the requesting agent continues without it. Set to 0 to disable the limit. + global: True + advanced: True + forcedType: int adapters: description: Configuration for AI adapters used by the Onion AI assistant. Please see documentation for help on which fields are required for which protocols. global: True From 8675296393aa3dd9b5f4f64177602546ec18cf00 Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Wed, 1 Jul 2026 15:04:42 -0600 Subject: [PATCH 02/28] More Agentic Fields The big agentic switch, a specific maxDelegationDepth, and the agentMapping dict --- salt/soc/defaults.yaml | 6 +++++- salt/soc/soc_soc.yaml | 11 +++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index afb4c37d1..c751b25f4 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1510,7 +1510,7 @@ soc: systemPromptAddendum: "" systemPromptAddendumMaxLength: 50000 maxSubSessionTokens: 0 - maxDelegationDepth: 0 + maxDelegationDepth: 5 adapters: - name: SOAI protocol: securityonion_ai_cloud @@ -1522,6 +1522,10 @@ soc: serviceAccountJSON: "" serviceAccountLocation: "" healthTimeoutSeconds: 5 + agentic: false + agentMapping: + Orchestrator: sonnet + Hunter: sonnet onionconfig: saltstackDir: /opt/so/saltstack bypassEnabled: false diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index cf483c406..a11f8e0e4 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -767,6 +767,17 @@ soc: label: Health Timeout Seconds required: False forcedType: int + agentic: + description: Indicates if the Assistant Module should operate in agentic mode or not. If true, agents can work together to solve tasks. + global: True + forcedType: bool + agentMapping: + Orchestrator: + description: The initial agent in most agentic conversations. This agent will delegate requests to specialized agents. + global: True + Hunter: + description: This agent is specialized in querying events. + global: True client: assistant: enabled: From 83cf1f0793efaf8ee446e216698eebb37bd842cb Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Thu, 2 Jul 2026 10:11:52 -0600 Subject: [PATCH 03/28] New Client Params for Tool Retries --- salt/soc/defaults.yaml | 2 ++ salt/soc/soc_soc.yaml | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index c751b25f4..490a09e3c 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -2695,6 +2695,8 @@ soc: thresholdColorRatioLow: 0.5 thresholdColorRatioMed: 0.75 thresholdColorRatioMax: 1 + toolBusyMaxRetries: 30 + toolBusyRetryDelayMs: 1000 availableModels: - id: sonnet displayName: Claude Sonnet diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index a11f8e0e4..5e171cd43 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -784,6 +784,12 @@ soc: description: Set to true to enable the Onion AI assistant in SOC. global: True forcedType: bool + toolBusyMaxRetries: + description: How many times to retry auto approving a tool while a tool is already running. + global: True + toolBusyRetryDelayMs: + description: How long in milliseconds to wait between each retry when auto approving a tool. + global: True investigationPrompt: description: Prompt given to Onion AI when beginning an investigation. global: True From ef83450107ce82935eb6fa46339e36a348160615 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 7 Jul 2026 16:54:06 -0400 Subject: [PATCH 04/28] recollate --- salt/manager/tools/sbin/soup | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 6725cc95c..f0be61546 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -789,7 +789,7 @@ bootstrap_so_soc_database() { # and runs automatically only on a fresh data directory. Hosts upgrading from # 3.1.0 already have /nsm/postgres populated, so the so_soc bootstrap block # added in 3.2 never fires. Re-run the script explicitly; it's idempotent. - echo "Bootstrapping so_soc database via init-db.sh." + echo "Bootstrapping database via init-db.sh." # The postgres image has no USER directive, so `docker exec` defaults to # root, and the container env intentionally omits POSTGRES_USER (the upstream # entrypoint defaults it transiently during first-init only). Recreate both @@ -800,10 +800,10 @@ bootstrap_so_soc_database() { return 0 fi if ! $exec_cmd; then - FINAL_MESSAGE_QUEUE+=("WARNING: init-db.sh failed inside so-postgres during the 3.2.0 upgrade; the so_soc database may not have been bootstrapped. Re-run manually: $exec_cmd") + FINAL_MESSAGE_QUEUE+=("WARNING: init-db.sh failed inside so-postgres during the 3.2.0 upgrade; the database may not have been bootstrapped. Re-run manually: $exec_cmd") return 0 fi - echo "so_soc bootstrap complete." + echo "Database bootstrap complete." } # Existing grids should keep ILM unless an admin explicitly opts in to DLM. @@ -873,6 +873,16 @@ update_kafka_metadata() { fi } +recollate_postgres() { + for db in postgres securityonion so-telegraf; do + exec_cmd="docker exec so-postgres psql -U postgres $db -c \"reindex database $db; alter database $db refresh collation version;\"" + if ! $exec_cmd; then + FINAL_MESSAGE_QUEUE+=("WARNING: recollating $db database failed. Re-run manually: $exec_cmd") + return 0 + fi + done +} + up_to_3.2.0() { fix_logstash_0013_lumberjack_pipeline_name @@ -884,6 +894,9 @@ up_to_3.2.0() { post_to_3.2.0() { bootstrap_so_soc_database + # Recollate due to image OS rebase + recollate_postgres + # Including agent regen script here since it was missed in post_to_3.1.0 echo "Regenerating Elastic Agent Installers" /sbin/so-elastic-agent-gen-installers From 57b7d59387269ab6a26c45bf7fc5e35cdad57d26 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Tue, 7 Jul 2026 16:42:47 -0500 Subject: [PATCH 05/28] verify elastic-agent reports healthy status before completing installation --- salt/elasticfleet/install_agent_grid.sls | 8 +- .../tools/sbin/so-elastic-agent-install | 75 +++++++++++++++++++ 2 files changed, 78 insertions(+), 5 deletions(-) create mode 100644 salt/elasticfleet/tools/sbin/so-elastic-agent-install diff --git a/salt/elasticfleet/install_agent_grid.sls b/salt/elasticfleet/install_agent_grid.sls index 482af2e1e..431aa6f97 100644 --- a/salt/elasticfleet/install_agent_grid.sls +++ b/salt/elasticfleet/install_agent_grid.sls @@ -21,11 +21,9 @@ pull_agent_installer: run_installer: cmd.run: - - name: ./so-elastic-agent_linux_amd64 -token={{ GRIDNODETOKEN }} -force - - cwd: /opt/so - - retry: - attempts: 3 - interval: 20 + - name: /usr/sbin/so-elastic-agent-install "{{ GRIDNODETOKEN }}" + - require: + - file: pull_agent_installer cleanup_agent_installer: file.absent: diff --git a/salt/elasticfleet/tools/sbin/so-elastic-agent-install b/salt/elasticfleet/tools/sbin/so-elastic-agent-install new file mode 100644 index 000000000..dda3f1672 --- /dev/null +++ b/salt/elasticfleet/tools/sbin/so-elastic-agent-install @@ -0,0 +1,75 @@ +#!/bin/bash +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +. /usr/sbin/so-elastic-fleet-common + + +# passed in as arg from elasticfleet/install_agent_grid.sls, else pulled from pillar later +GRIDNODETOKEN="$1" + +check_agent_health() { + timeout=180 + interval=10 + start=$SECONDS + + while (( SECONDS - start < timeout )); do + if elastic-agent status 2>/dev/null | grep -A1 'elastic-agent$' | grep -q 'status: (HEALTHY)'; then + return 0 + fi + echo "elastic-agent is not yet healthy. Waiting for ${interval} seconds before checking again..." + sleep "$interval" + done + + echo "elastic-agent did not become healthy within ${timeout} seconds" + return 1 +} + +if command -v elastic-agent >/dev/null 2>&1; then + elastic-agent uninstall -f +fi + +if [[ -z "$GRIDNODETOKEN" ]]; then + noderole=$(so-yaml.py get -r /etc/salt/grains role) + if [[ "$noderole" == "so-heavynode" ]]; then + GRIDNODETOKEN=$(salt-call pillar.get global:fleet_grid_enrollment_token_heavy --out=newline_values_only) + else + GRIDNODETOKEN=$(salt-call pillar.get global:fleet_grid_enrollment_token_general --out=newline_values_only) + fi +fi + +if [[ ! -x /opt/so/so-elastic-agent_linux_amd64 ]]; then + echo "Downloading elastic-agent installer... This could take a while if another Salt job is running." + salt-call state.sls_id pull_agent_installer elasticfleet.install_agent_grid queue=True +fi + +if [[ -x /opt/so/so-elastic-agent_linux_amd64 ]]; then + attempts=0 + cd /opt/so/ || exit 1 + + while [[ $attempts -lt 3 ]]; do + if ./so-elastic-agent_linux_amd64 -token="$GRIDNODETOKEN" -force; echo "Verifying elastic-agent health..." && check_agent_health; then + rm -f /opt/so/so-elastic-agent_linux_amd64 + elastic-agent status + + exit 0 + fi + + attempts=$((attempts + 1)) + + if [[ $attempts -lt 3 ]]; then + echo "so-elastic-agent installer failed. Retrying in 20 seconds..." + sleep 20 + fi + done + + echo "so-elastic-agent installer failed after 3 attempts. Exiting." + + exit 1 +else + echo "Unable to locate so-elastic-agent installer. Exiting." + + exit 1 +fi From 70af3cec534dfa2c5122893ce73ed1c9b29632d4 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Tue, 7 Jul 2026 16:48:10 -0500 Subject: [PATCH 06/28] avoid using 'failure' until all loops are done so so-verify doesn't flag it --- .../elasticfleet/tools/sbin/so-elastic-agent-install | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/salt/elasticfleet/tools/sbin/so-elastic-agent-install b/salt/elasticfleet/tools/sbin/so-elastic-agent-install index dda3f1672..fd6bceeff 100644 --- a/salt/elasticfleet/tools/sbin/so-elastic-agent-install +++ b/salt/elasticfleet/tools/sbin/so-elastic-agent-install @@ -19,11 +19,11 @@ check_agent_health() { if elastic-agent status 2>/dev/null | grep -A1 'elastic-agent$' | grep -q 'status: (HEALTHY)'; then return 0 fi - echo "elastic-agent is not yet healthy. Waiting for ${interval} seconds before checking again..." + echo "The Elastic Agent is not yet healthy. Waiting for ${interval} seconds before checking again..." sleep "$interval" done - echo "elastic-agent did not become healthy within ${timeout} seconds" + echo "The Elastic Agent did not become healthy within ${timeout} seconds" return 1 } @@ -41,7 +41,7 @@ if [[ -z "$GRIDNODETOKEN" ]]; then fi if [[ ! -x /opt/so/so-elastic-agent_linux_amd64 ]]; then - echo "Downloading elastic-agent installer... This could take a while if another Salt job is running." + echo "Downloading so-elastic-agent installer... This could take a while if another Salt job is running." salt-call state.sls_id pull_agent_installer elasticfleet.install_agent_grid queue=True fi @@ -50,7 +50,7 @@ if [[ -x /opt/so/so-elastic-agent_linux_amd64 ]]; then cd /opt/so/ || exit 1 while [[ $attempts -lt 3 ]]; do - if ./so-elastic-agent_linux_amd64 -token="$GRIDNODETOKEN" -force; echo "Verifying elastic-agent health..." && check_agent_health; then + if ./so-elastic-agent_linux_amd64 -token="$GRIDNODETOKEN" -force; echo "Verifying Elastic Agent health..." && check_agent_health; then rm -f /opt/so/so-elastic-agent_linux_amd64 elastic-agent status @@ -60,12 +60,12 @@ if [[ -x /opt/so/so-elastic-agent_linux_amd64 ]]; then attempts=$((attempts + 1)) if [[ $attempts -lt 3 ]]; then - echo "so-elastic-agent installer failed. Retrying in 20 seconds..." + echo "Unable to verify Elastic Agent health... Retrying in 20 seconds..." sleep 20 fi done - echo "so-elastic-agent installer failed after 3 attempts. Exiting." + echo "The so-elastic-agent installer failed after 3 attempts. Exiting." exit 1 else From 7f6014096b199e4aa38bc36fa17e4433ee9aa20e Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 7 Jul 2026 22:01:27 -0400 Subject: [PATCH 07/28] recollate db --- salt/manager/tools/sbin/soup | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index f0be61546..9264d9dca 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -874,12 +874,9 @@ update_kafka_metadata() { } recollate_postgres() { - for db in postgres securityonion so-telegraf; do - exec_cmd="docker exec so-postgres psql -U postgres $db -c \"reindex database $db; alter database $db refresh collation version;\"" - if ! $exec_cmd; then - FINAL_MESSAGE_QUEUE+=("WARNING: recollating $db database failed. Re-run manually: $exec_cmd") - return 0 - fi + for db in postgres securityonion so_telegraf; do + docker exec so-postgres psql -U postgres $db -c "reindex database $db" + docker exec so-postgres psql -U postgres $db -c "alter database $db refresh collation version" done } From 8a8f2c4a336a103ea90449414b0fa16cb8b2d962 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 7 Jul 2026 22:22:00 -0400 Subject: [PATCH 08/28] change order to recollate first --- salt/manager/tools/sbin/soup | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 9264d9dca..56f4c66dd 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -889,11 +889,11 @@ up_to_3.2.0() { } post_to_3.2.0() { - bootstrap_so_soc_database - # Recollate due to image OS rebase recollate_postgres + bootstrap_so_soc_database + # Including agent regen script here since it was missed in post_to_3.1.0 echo "Regenerating Elastic Agent Installers" /sbin/so-elastic-agent-gen-installers From d131d167de8e2b5633b84076a8f9116497185dcd Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 8 Jul 2026 07:00:38 -0400 Subject: [PATCH 09/28] provide explanation text --- salt/manager/tools/sbin/soup | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 56f4c66dd..aee767db7 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -874,10 +874,14 @@ update_kafka_metadata() { } recollate_postgres() { + echo "" + echo "Recollating PostgreSQL databases. The following output may contain warnings about a version mismatch, followed by a note indicating that the collation version has been changed." for db in postgres securityonion so_telegraf; do docker exec so-postgres psql -U postgres $db -c "reindex database $db" docker exec so-postgres psql -U postgres $db -c "alter database $db refresh collation version" done + echo "Recollating PostgreSQL databases complete." + echo "" } up_to_3.2.0() { From f8de176f4bbbde8b8fd5f6aae06b48c93d1709b1 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Wed, 8 Jul 2026 07:47:47 -0500 Subject: [PATCH 10/28] uninstall agent on final failed attempt --- .../tools/sbin/so-elastic-agent-install | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/salt/elasticfleet/tools/sbin/so-elastic-agent-install b/salt/elasticfleet/tools/sbin/so-elastic-agent-install index fd6bceeff..920b17137 100644 --- a/salt/elasticfleet/tools/sbin/so-elastic-agent-install +++ b/salt/elasticfleet/tools/sbin/so-elastic-agent-install @@ -27,9 +27,13 @@ check_agent_health() { return 1 } -if command -v elastic-agent >/dev/null 2>&1; then - elastic-agent uninstall -f -fi +uninstall_agent() { + if command -v elastic-agent >/dev/null 2>&1; then + elastic-agent uninstall -f + fi +} + +uninstall_agent if [[ -z "$GRIDNODETOKEN" ]]; then noderole=$(so-yaml.py get -r /etc/salt/grains role) @@ -40,6 +44,12 @@ if [[ -z "$GRIDNODETOKEN" ]]; then fi fi +if [[ -z "$GRIDNODETOKEN" ]]; then + echo "Unable to determine Elastic Fleet enrollment token. Exiting." + + exit 1 +fi + if [[ ! -x /opt/so/so-elastic-agent_linux_amd64 ]]; then echo "Downloading so-elastic-agent installer... This could take a while if another Salt job is running." salt-call state.sls_id pull_agent_installer elasticfleet.install_agent_grid queue=True @@ -50,7 +60,7 @@ if [[ -x /opt/so/so-elastic-agent_linux_amd64 ]]; then cd /opt/so/ || exit 1 while [[ $attempts -lt 3 ]]; do - if ./so-elastic-agent_linux_amd64 -token="$GRIDNODETOKEN" -force; echo "Verifying Elastic Agent health..." && check_agent_health; then + if ./so-elastic-agent_linux_amd64 -token="$GRIDNODETOKEN" -force && echo "Verifying Elastic Agent health..." && check_agent_health; then rm -f /opt/so/so-elastic-agent_linux_amd64 elastic-agent status @@ -65,6 +75,8 @@ if [[ -x /opt/so/so-elastic-agent_linux_amd64 ]]; then fi done + uninstall_agent + rm -f /opt/so/so-elastic-agent_linux_amd64 echo "The so-elastic-agent installer failed after 3 attempts. Exiting." exit 1 From 2a4a7307f70d2b39229b3abb545d80554999a51f Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Wed, 8 Jul 2026 07:51:13 -0500 Subject: [PATCH 11/28] uninstall agent after downloading new one and getting gridtoken --- salt/elasticfleet/tools/sbin/so-elastic-agent-install | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/elasticfleet/tools/sbin/so-elastic-agent-install b/salt/elasticfleet/tools/sbin/so-elastic-agent-install index 920b17137..bda8306f0 100644 --- a/salt/elasticfleet/tools/sbin/so-elastic-agent-install +++ b/salt/elasticfleet/tools/sbin/so-elastic-agent-install @@ -33,8 +33,6 @@ uninstall_agent() { fi } -uninstall_agent - if [[ -z "$GRIDNODETOKEN" ]]; then noderole=$(so-yaml.py get -r /etc/salt/grains role) if [[ "$noderole" == "so-heavynode" ]]; then @@ -55,6 +53,8 @@ if [[ ! -x /opt/so/so-elastic-agent_linux_amd64 ]]; then salt-call state.sls_id pull_agent_installer elasticfleet.install_agent_grid queue=True fi +uninstall_agent + if [[ -x /opt/so/so-elastic-agent_linux_amd64 ]]; then attempts=0 cd /opt/so/ || exit 1 From 85d7f6bebcf406fde9425171b12a1b5f182ac076 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Wed, 8 Jul 2026 08:19:15 -0500 Subject: [PATCH 12/28] independently download so-elastic-agent installer --- .../tools/sbin/so-elastic-agent-install | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/salt/elasticfleet/tools/sbin/so-elastic-agent-install b/salt/elasticfleet/tools/sbin/so-elastic-agent-install index bda8306f0..c6a499e94 100644 --- a/salt/elasticfleet/tools/sbin/so-elastic-agent-install +++ b/salt/elasticfleet/tools/sbin/so-elastic-agent-install @@ -50,15 +50,23 @@ fi if [[ ! -x /opt/so/so-elastic-agent_linux_amd64 ]]; then echo "Downloading so-elastic-agent installer... This could take a while if another Salt job is running." - salt-call state.sls_id pull_agent_installer elasticfleet.install_agent_grid queue=True -fi -uninstall_agent + # When running outside of elasticfleet/install_agent_grid.sls we need to download the installer independently. + # PYTHONWARNINGS="ignore" to avoid messages like the following when running salt-call: + # '/opt/saltstack/salt/lib/python3.10/site-packages/salt/transport/base.py:129: TransportWarning: Unclosed transport! + # File "/bin/salt-call", line 12, in + # sys.exit(salt_call())' + + PYTHONWARNINGS="ignore" salt-call state.single file.managed name=/opt/so/so-elastic-agent_linux_amd64 source=salt://elasticfleet/files/so_agent-installers/so-elastic-agent_linux_amd64 mode=755 makedirs=True queue=True + +fi if [[ -x /opt/so/so-elastic-agent_linux_amd64 ]]; then attempts=0 cd /opt/so/ || exit 1 + uninstall_agent + while [[ $attempts -lt 3 ]]; do if ./so-elastic-agent_linux_amd64 -token="$GRIDNODETOKEN" -force && echo "Verifying Elastic Agent health..." && check_agent_health; then rm -f /opt/so/so-elastic-agent_linux_amd64 From 1f44e9868102ddabf5fee09d73c45615c53df90a Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 8 Jul 2026 14:34:27 -0400 Subject: [PATCH 13/28] restart soc after re-initing db --- salt/manager/tools/sbin/soup | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index aee767db7..e33c27f6f 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -784,6 +784,17 @@ post_to_3.1.0() { ### 3.2.0 Scripts ### +recollate_postgres() { + echo "" + echo "Recollating PostgreSQL databases. The following output may contain warnings about a version mismatch, followed by a note indicating that the collation version has been changed." + for db in postgres securityonion so_telegraf; do + docker exec so-postgres psql -U postgres $db -c "reindex database $db" + docker exec so-postgres psql -U postgres $db -c "alter database $db refresh collation version" + done + echo "Recollating PostgreSQL databases complete." + echo "" +} + bootstrap_so_soc_database() { # init-db.sh is mounted into so-postgres at /docker-entrypoint-initdb.d/init-db.sh # and runs automatically only on a fresh data directory. Hosts upgrading from @@ -804,6 +815,9 @@ bootstrap_so_soc_database() { return 0 fi echo "Database bootstrap complete." + + echo "Restarting so-soc container to pick up database changes" + docker restart so-soc } # Existing grids should keep ILM unless an admin explicitly opts in to DLM. @@ -873,17 +887,6 @@ update_kafka_metadata() { fi } -recollate_postgres() { - echo "" - echo "Recollating PostgreSQL databases. The following output may contain warnings about a version mismatch, followed by a note indicating that the collation version has been changed." - for db in postgres securityonion so_telegraf; do - docker exec so-postgres psql -U postgres $db -c "reindex database $db" - docker exec so-postgres psql -U postgres $db -c "alter database $db refresh collation version" - done - echo "Recollating PostgreSQL databases complete." - echo "" -} - up_to_3.2.0() { fix_logstash_0013_lumberjack_pipeline_name From 5a57bbe4de6bb60e7c55ebfa3b44cdad5bd2c308 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Wed, 8 Jul 2026 14:03:31 -0500 Subject: [PATCH 14/28] add missing so-logs-soc annotation --- salt/elasticsearch/soc_elasticsearch.yaml | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index 46e836c9c..95880bb9d 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -28,14 +28,14 @@ elasticsearch: description: The maximum number of memory map areas a process may use. Elasticsearch uses a mmapfs directory by default to store its indices. The default operating system limits on mmap counts could be too low, which may result in out of memory exceptions. forcedType: int helpLink: elasticsearch - retention: + retention: retention_pct: decription: Total percentage of space used by Elasticsearch for multi node clusters helpLink: elasticsearch global: True config: cluster: - name: + name: description: The name of the Security Onion Elasticsearch cluster, for identification purposes. readonly: True global: True @@ -55,13 +55,13 @@ elasticsearch: forcedType: bool helpLink: elasticsearch watermark: - low: + low: description: The lower percentage of used disk space representing a healthy node. helpLink: elasticsearch - high: + high: description: The higher percentage of used disk space representing an unhealthy node. helpLink: elasticsearch - flood_stage: + flood_stage: description: The max percentage of used disk space that will cause the node to take protective actions, such as blocking incoming events. helpLink: elasticsearch action: @@ -172,11 +172,11 @@ elasticsearch: forcedType: int global: True helpLink: elasticsearch - refresh_interval: + refresh_interval: description: Seconds between index refreshes. Shorter intervals can cause query performance to suffer since this is a synchronous and resource-intensive operation. global: True helpLink: elasticsearch - number_of_shards: + number_of_shards: description: Number of shards required for this index. Using multiple shards increases fault tolerance, but also increases storage and network costs. global: True helpLink: elasticsearch @@ -269,7 +269,7 @@ elasticsearch: global: True advanced: True warm: - min_age: + min_age: description: Minimum age of index. ex. 30d - This determines when the index should be moved to the warm tier. Nodes in the warm tier generally don’t need to be as fast as those in the hot tier. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and warm min_age set to 30 then there will be 30 days from index creation to rollover and then an additional 30 days before moving to warm tier. regex: ^[0-9]{1,5}d$ forcedType: string @@ -370,7 +370,7 @@ elasticsearch: template: settings: index: - number_of_replicas: + number_of_replicas: description: Number of replicas required for this index. Multiple replicas protects against data loss, but also increases storage costs. forcedType: int global: True @@ -391,12 +391,12 @@ elasticsearch: global: True advanced: True helpLink: elasticsearch - refresh_interval: + refresh_interval: description: Seconds between index refreshes. Shorter intervals can cause query performance to suffer since this is a synchronous and resource-intensive operation. global: True advanced: True helpLink: elasticsearch - number_of_shards: + number_of_shards: description: Number of shards required for this index. Using multiple shards increases fault tolerance, but also increases storage and network costs. global: True advanced: True @@ -645,6 +645,7 @@ elasticsearch: global: True advanced: True helpLink: elasticsearch + so-logs-soc: *dataStreamSettings so-logs-system_x_auth: *dataStreamSettings so-logs-system_x_syslog: *dataStreamSettings so-logs-system_x_system: *dataStreamSettings From c04a30785f06218510b3d37a917ab9e02e21b977 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Wed, 8 Jul 2026 16:15:02 -0500 Subject: [PATCH 15/28] make elastic agent state persistent to prevent re-enrollment during soup / reboots --- salt/common/tools/sbin/so-restart | 3 +++ salt/common/tools/sbin/so-stop | 2 ++ salt/elasticfleet/enabled.sls | 10 ++++++++++ 3 files changed, 15 insertions(+) diff --git a/salt/common/tools/sbin/so-restart b/salt/common/tools/sbin/so-restart index 14747d134..b3864d2d1 100755 --- a/salt/common/tools/sbin/so-restart +++ b/salt/common/tools/sbin/so-restart @@ -35,6 +35,9 @@ case $1 in "elastic-fleet"|"elasticfleet") docker_check_running "elastic-fleet" "--stop" docker rm "so-elastic-fleet" 2> /dev/null + # Removing the elastic fleet state directory, so that the next startup re-enrolls with a fresh policy + rm -rf /opt/so/conf/elastic-fleet/state + salt-call state.apply elasticfleet queue=True ;; *) diff --git a/salt/common/tools/sbin/so-stop b/salt/common/tools/sbin/so-stop index d036a7b63..9218b44f7 100755 --- a/salt/common/tools/sbin/so-stop +++ b/salt/common/tools/sbin/so-stop @@ -29,6 +29,8 @@ case $1 in "elasticfleet"|"elastic-fleet") docker_check_running "elastic-fleet" "--stop" docker rm "so-elastic-fleet" 2> /dev/null + # Removing the elastic fleet state directory, so that the next startup re-enrolls with a fresh policy + rm -rf /opt/so/conf/elastic-fleet/state ;; *) docker_check_running "$1" "--stop" diff --git a/salt/elasticfleet/enabled.sls b/salt/elasticfleet/enabled.sls index 60cf27deb..ea64a4cb1 100644 --- a/salt/elasticfleet/enabled.sls +++ b/salt/elasticfleet/enabled.sls @@ -11,6 +11,10 @@ {# This value is generated during node install and stored in minion pillar #} {% set SERVICETOKEN = salt['pillar.get']('elasticfleet:config:server:es_token','') %} +{# Prevent Elastic Agent from re-enrolling with a new agent.id everytime the container starts up. + - if a fresh enrollment is needed use 'so-stop elasticfleet' + #} +{% set ENROLLED = salt['file.file_exists']('/opt/so/conf/elastic-fleet/state/fleet.enc') %} include: - ca @@ -65,6 +69,7 @@ so-elastic-fleet: - /etc/pki/elasticfleet-server.crt:/etc/pki/elasticfleet-server.crt:ro - /etc/pki/elasticfleet-server.key:/etc/pki/elasticfleet-server.key:ro - /etc/pki/tls/certs/intca.crt:/etc/pki/tls/certs/intca.crt:ro + - /opt/so/conf/elastic-fleet/state:/usr/share/elastic-agent/state - /opt/so/log/elasticfleet:/usr/share/elastic-agent/logs {% if DOCKERMERGED.containers['so-elastic-fleet'].custom_bind_mounts %} {% for BIND in DOCKERMERGED.containers['so-elastic-fleet'].custom_bind_mounts %} @@ -72,6 +77,7 @@ so-elastic-fleet: {% endfor %} {% endif %} - environment: + {% if not ENROLLED %} - FLEET_SERVER_ENABLE=true - FLEET_URL=https://{{ GLOBALS.hostname }}:8220 - FLEET_SERVER_ELASTICSEARCH_HOST=https://{{ GLOBALS.manager }}:9200 @@ -81,6 +87,9 @@ so-elastic-fleet: - FLEET_SERVER_CERT_KEY=/etc/pki/elasticfleet-server.key - FLEET_CA=/etc/pki/tls/certs/intca.crt - FLEET_SERVER_ELASTICSEARCH_CA=/etc/pki/tls/certs/intca.crt + {% endif %} + - STATE_PATH=/usr/share/elastic-agent/state + - CONFIG_PATH=/usr/share/elastic-agent/state - LOGS_PATH=logs {% if DOCKERMERGED.containers['so-elastic-fleet'].extra_env %} {% for XTRAENV in DOCKERMERGED.containers['so-elastic-fleet'].extra_env %} @@ -99,6 +108,7 @@ so-elastic-fleet: - x509: etc_elasticfleet_crt - require: - file: trusttheca + - file: eastatedir - x509: etc_elasticfleet_key - x509: etc_elasticfleet_crt From 3394e9aab78bb832392682593b1a05c2c45190cf Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Thu, 9 Jul 2026 11:29:43 -0500 Subject: [PATCH 16/28] increase agent health timeout and add logging for elastic agent status output --- salt/elasticfleet/tools/sbin/so-elastic-agent-install | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/salt/elasticfleet/tools/sbin/so-elastic-agent-install b/salt/elasticfleet/tools/sbin/so-elastic-agent-install index c6a499e94..93889c002 100644 --- a/salt/elasticfleet/tools/sbin/so-elastic-agent-install +++ b/salt/elasticfleet/tools/sbin/so-elastic-agent-install @@ -9,14 +9,19 @@ # passed in as arg from elasticfleet/install_agent_grid.sls, else pulled from pillar later GRIDNODETOKEN="$1" +LOGFILE="/opt/so/SO-Elastic-Agent_Installer_Health.log" check_agent_health() { - timeout=180 + timeout=300 interval=10 start=$SECONDS + truncate -s 0 "$LOGFILE" + while (( SECONDS - start < timeout )); do - if elastic-agent status 2>/dev/null | grep -A1 'elastic-agent$' | grep -q 'status: (HEALTHY)'; then + agent_status=$(elastic-agent status 2>&1) + echo -e "\n$(date)\n$agent_status\n" >> "$LOGFILE" + if echo "$agent_status" | grep -A1 'elastic-agent$' | grep -q 'status: (HEALTHY)'; then return 0 fi echo "The Elastic Agent is not yet healthy. Waiting for ${interval} seconds before checking again..." From 6fa0d327cbafd9a18d38a4f3bb51b1f145e74cfe Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Thu, 9 Jul 2026 14:14:58 -0500 Subject: [PATCH 17/28] cause so-setup to fail if there are issues setting up fleet --- .../tools/sbin_jinja/so-elastic-fleet-setup | 28 +++++++++++++++---- 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup index b62310375..2b677606a 100755 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup @@ -244,11 +244,27 @@ printf '%s\n'\ "" >> "$global_pillar_file" # Call Elastic-Fleet Salt State -printf "\nApplying elasticfleet state" -salt-call state.apply elasticfleet queue=True +printf "\nApplying elasticfleet state\n" +if ! salt-call state.apply elasticfleet queue=True; then + printf "\nFailure(s) in elasticfleet state... Exiting...\n" + exit 1 +fi # Generate installers & install Elastic Agent on the node -so-elastic-agent-gen-installers -printf "\nApplying elasticfleet.install_agent_grid state" -salt-call state.apply elasticfleet.install_agent_grid queue=True -exit 0 +for agent_gen_attempt in {1..3}; do + if so-elastic-agent-gen-installers; then + break + elif [[ $agent_gen_attempt -lt 3 ]]; then + printf "\nUnable to generate Elastic Agent installers... Attempt (%s/3). Retrying...\n" "$agent_gen_attempt" + sleep 10 + else + printf "\nFailed to generate Elastic Agent installers after 3 attempts. Exiting...\n" + exit 1 + fi +done + +printf "\nApplying elasticfleet.install_agent_grid state\n" +if ! salt-call state.apply elasticfleet.install_agent_grid queue=True; then + printf "\nFailure(s) in elasticfleet.install_agent_grid state... Exiting...\n" + exit 1 +fi \ No newline at end of file From 8b0759866ea8d2c121c0eb32b9bdf581fed6c7e8 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Thu, 9 Jul 2026 14:15:22 -0500 Subject: [PATCH 18/28] verify installers are generated --- .../so-elastic-agent-gen-installers | 39 +++++++++++++++---- 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers index d25c18e29..de342657f 100755 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers @@ -30,7 +30,7 @@ done if [[ -z $FLEETHOST ]] || [[ -z $ENROLLMENTOKEN ]]; then printf "\nFleet Host URL, Enrollment Token or Elastic Version empty - exiting..." printf "\nFleet Host: $FLEETHOST, Enrollment Token: $ENROLLMENTOKEN\n" - exit + exit 1 fi OSARCH=( "linux-x86_64" "windows-x86_64" "darwin-x86_64" "darwin-aarch64" ) @@ -62,31 +62,54 @@ do done GOTARGETOS=( "linux" "windows" "darwin" "darwin/arm64" ) -GOARCH="amd64" printf "\n### Generating OS packages using the cleaned up tarballs" -for GOOS in "${GOTARGETOS[@]}" -do +for GOOS in "${GOTARGETOS[@]}"; do + GOARCH="amd64" if [[ $GOOS == 'darwin/arm64' ]]; then GOOS="darwin" && GOARCH="arm64"; fi printf "\n\n### Generating $GOOS/$GOARCH Installer...\n" docker run -e CGO_ENABLED=0 -e GOOS=$GOOS -e GOARCH=$GOARCH \ --mount type=bind,source=/etc/pki/tls/certs/,target=/workspace/files/cert/ \ --mount type=bind,source=/nsm/elastic-agent-workspace/,target=/workspace/files/elastic-agent/ \ - --mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/,target=/output/ \ + --mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/,target=/output/ \ {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} go build -ldflags "-X main.fleetHostURLsList=$FLEETHOST -X main.enrollmentToken=$ENROLLMENTOKEN" -o /output/so-elastic-agent_${GOOS}_${GOARCH} printf "\n### $GOOS/$GOARCH Installer Generated...\n" done printf "\n\n### Generating MSI...\n" -cp /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/so-elastic-agent_windows_amd64 /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/so-elastic-agent_windows_amd64.exe +cp /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_windows_amd64 /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_windows_amd64.exe docker run \ ---mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/,target=/output/ -w /output \ +--mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/,target=/output/ -w /output \ {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} wixl -o so-elastic-agent_windows_amd64_msi --arch x64 /workspace/so-elastic-agent.wxs printf "\n### MSI Generated...\n" +# Verify installers were created +for GOOS in "${GOTARGETOS[@]}"; do + GOARCH="amd64" + if [[ $GOOS == 'darwin/arm64' ]]; then GOOS="darwin"; GOARCH="arm64"; fi + if [[ ! -f /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_${GOOS}_${GOARCH} ]]; then + printf "\n### ERROR: Installer for %s/%s was not generated. Exiting...\n" "$GOOS" "$GOARCH" + exit 1 + fi + # After verifying new installer was generated, move it to so_agent-installers directory + mv /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_${GOOS}_${GOARCH} /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/ +done + +# Verify MSI installer +if [[ ! -f /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_windows_amd64_msi ]]; then + printf "\n### ERROR: Installer MSI was not generated. Exiting...\n" + exit 1 +else + # After verifying new installer MSI was generated, move it to so_agent-installers directory + mv /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_windows_amd64_msi /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/ +fi + printf "\n### Cleaning up temp files \n" rm -rf /nsm/elastic-agent-workspace -rm -rf /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/so-elastic-agent_windows_amd64.exe +rm -rf /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_windows_amd64.exe printf "\n### Copying so_agent-installers to /nsm/elastic-fleet/ for nginx.\n" \cp -vr /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/ /nsm/elastic-fleet/ chmod 644 /nsm/elastic-fleet/so_agent-installers/* + +# if we got here all installers have been generated successfully +exit 0 From 2959dc95644a4372dadd425fd72929d32e093177 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Thu, 9 Jul 2026 14:18:08 -0500 Subject: [PATCH 19/28] keep logs for all 3 attempts --- salt/elasticfleet/tools/sbin/so-elastic-agent-install | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/elasticfleet/tools/sbin/so-elastic-agent-install b/salt/elasticfleet/tools/sbin/so-elastic-agent-install index 93889c002..8f57d217c 100644 --- a/salt/elasticfleet/tools/sbin/so-elastic-agent-install +++ b/salt/elasticfleet/tools/sbin/so-elastic-agent-install @@ -16,8 +16,6 @@ check_agent_health() { interval=10 start=$SECONDS - truncate -s 0 "$LOGFILE" - while (( SECONDS - start < timeout )); do agent_status=$(elastic-agent status 2>&1) echo -e "\n$(date)\n$agent_status\n" >> "$LOGFILE" @@ -70,6 +68,8 @@ if [[ -x /opt/so/so-elastic-agent_linux_amd64 ]]; then attempts=0 cd /opt/so/ || exit 1 + truncate -s 0 "$LOGFILE" + uninstall_agent while [[ $attempts -lt 3 ]]; do From 0b078c48041a30947efbb6dc8bf4945e26272339 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Thu, 9 Jul 2026 14:23:45 -0500 Subject: [PATCH 20/28] give the elasticfleet state a few chances to complete successfully before exiting 1 causing so-setup to fail --- .../tools/sbin_jinja/so-elastic-fleet-setup | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup index 2b677606a..21d74cf6f 100755 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup @@ -245,10 +245,17 @@ printf '%s\n'\ # Call Elastic-Fleet Salt State printf "\nApplying elasticfleet state\n" -if ! salt-call state.apply elasticfleet queue=True; then - printf "\nFailure(s) in elasticfleet state... Exiting...\n" - exit 1 -fi +for state_attempt in {1..3}; do + if salt-call state.apply elasticfleet queue=True; then + break + elif [[ $state_attempt -lt 3 ]]; then + printf "\nElasticfleet state did not complete successfully... Attempt (%s/3). Retrying...\n" "$state_attempt" + sleep 10 + else + printf "\nFailure(s) in elasticfleet state... Exiting...\n" + exit 1 + fi +done # Generate installers & install Elastic Agent on the node for agent_gen_attempt in {1..3}; do From f6a2758321891c9e12f0d4efe29a76ee446067e5 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Thu, 9 Jul 2026 14:44:42 -0500 Subject: [PATCH 21/28] add so-elastic-agent-install script to minions --- salt/elasticfleet/install_agent_grid.sls | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/salt/elasticfleet/install_agent_grid.sls b/salt/elasticfleet/install_agent_grid.sls index 431aa6f97..7ed727b48 100644 --- a/salt/elasticfleet/install_agent_grid.sls +++ b/salt/elasticfleet/install_agent_grid.sls @@ -10,6 +10,15 @@ {% set AGENT_STATUS = salt['service.available']('elastic-agent') %} {% set AGENT_EXISTS = salt['file.file_exists']('/opt/Elastic/Agent/elastic-agent') %} +so-elastic-agent-install: + file.managed: + - name: /usr/sbin/so-elastic-agent-install + - source: salt://elasticfleet/tools/sbin/so-elastic-agent-install + - user: 947 + - group: 939 + - mode: 755 + - show_changes: False + {% if not AGENT_STATUS or not AGENT_EXISTS %} pull_agent_installer: From 566f90a0c0e544260ce94349553099a90c357411 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 10 Jul 2026 10:32:25 -0400 Subject: [PATCH 22/28] toggle pg metrics --- salt/soc/defaults.map.jinja | 24 +++++++++++++++++------- salt/soc/merged.map.jinja | 19 +++++++++++++++++++ 2 files changed, 36 insertions(+), 7 deletions(-) diff --git a/salt/soc/defaults.map.jinja b/salt/soc/defaults.map.jinja index 2821bb8e5..93fb63efe 100644 --- a/salt/soc/defaults.map.jinja +++ b/salt/soc/defaults.map.jinja @@ -8,6 +8,7 @@ {% from 'docker/docker.map.jinja' import DOCKERMERGED -%} {% set INFLUXDB_TOKEN = salt['pillar.get']('influxdb:token') %} {% import_text 'influxdb/metrics_link.txt' as METRICS_LINK %} +{% from 'telegraf/map.jinja' import TELEGRAFMERGED %} {% for module, application_url in GLOBALS.application_urls.items() %} {% do SOCDEFAULTS.soc.config.server.modules[module].update({'hostUrl': application_url}) %} @@ -24,13 +25,22 @@ {% do SOCDEFAULTS.soc.config.server.modules.elastic.update({'username': GLOBALS.elasticsearch.auth.users.so_elastic_user.user, 'password': GLOBALS.elasticsearch.auth.users.so_elastic_user.pass}) %} -{% do SOCDEFAULTS.soc.config.server.modules.influxdb.update({'hostUrl': 'https://' ~ GLOBALS.influxdb_host ~ ':8086'}) %} -{% do SOCDEFAULTS.soc.config.server.modules.influxdb.update({'token': INFLUXDB_TOKEN}) %} -{% for tool in SOCDEFAULTS.soc.config.server.client.tools %} -{% if tool.name == "toolInfluxDb" and METRICS_LINK | length > 0 %} -{% do tool.update({'link': METRICS_LINK}) %} -{% endif %} -{% endfor %} +{% if TELEGRAFMERGED.output == 'POSTGRES' %} +{% for tool in SOCDEFAULTS.soc.config.server.client.tools %} +{% if tool.name == "toolInfluxDb" %} +{% do SOCDEFAULTS.soc.config.server.client.tools.remove(tool) %} +{% endif %} +{% endfor %} + +{% else %} +{% do SOCDEFAULTS.soc.config.server.modules.influxdb.update({'hostUrl': 'https://' ~ GLOBALS.influxdb_host ~ ':8086'}) %} +{% do SOCDEFAULTS.soc.config.server.modules.influxdb.update({'token': INFLUXDB_TOKEN}) %} +{% for tool in SOCDEFAULTS.soc.config.server.client.tools %} +{% if tool.name == "toolInfluxDb" and METRICS_LINK | length > 0 %} +{% do tool.update({'link': METRICS_LINK}) %} +{% endif %} +{% endfor %} +{% endif %} {% do SOCDEFAULTS.soc.config.server.modules.statickeyauth.update({'anonymousCidr': DOCKERMERGED.range, 'apiKey': pillar.sensoroni.config.sensoronikey}) %} diff --git a/salt/soc/merged.map.jinja b/salt/soc/merged.map.jinja index cfc0fafbd..a421711e2 100644 --- a/salt/soc/merged.map.jinja +++ b/salt/soc/merged.map.jinja @@ -7,6 +7,11 @@ {% from 'soc/defaults.map.jinja' import SOCDEFAULTS with context %} {% from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_NODES %} {% from 'manager/map.jinja' import MANAGERMERGED %} +{% from 'telegraf/map.jinja' import TELEGRAFMERGED %} +{%- set PG_ENTRY = salt['pillar.get']('telegraf:postgres_creds:' ~ grains.id, {}) %} +{%- set PG_USER = PG_ENTRY.get('user', '') %} +{%- set PG_PASS = PG_ENTRY.get('pass', '') %} + {% set DOCKER_EXTRA_HOSTS = ELASTICSEARCH_NODES %} {% do DOCKER_EXTRA_HOSTS.append({GLOBALS.influxdb_host:pillar.node_data[GLOBALS.influxdb_host].ip}) %} @@ -75,6 +80,20 @@ {% do SOCMERGED.config.server.update({'airgapEnabled': false}) %} {% endif %} +{# Define the postgresmetrics module if telegraf is setup to only use Postgres #} +{% if TELEGRAFMERGED.output != 'INFLUXDB' and PG_USER and PG_PASS %} +{% do SOCMERGED.config.server.modules.update({ + 'postgresmetrics': { + 'database': 'so_telegraf', + 'host': GLOBALS.manager_ip, + 'password': PG_PASS, + 'port': 5432, + 'sslMode': 'allow', + 'user': PG_USER, + } + }) %} +{% do SOCMERGED.config.server.modules.pop('influxdb') %} +{% endif %} {# Define the Detections custom ruleset that should always be present #} {% set CUSTOM_RULESET = { From 6fc0fd954c7562e94fbf751dbea8bab98564ad9b Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 10 Jul 2026 10:35:37 -0400 Subject: [PATCH 23/28] clarify output annotation --- salt/telegraf/soc_telegraf.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/telegraf/soc_telegraf.yaml b/salt/telegraf/soc_telegraf.yaml index 4b9a2e3d1..0decdd06a 100644 --- a/salt/telegraf/soc_telegraf.yaml +++ b/salt/telegraf/soc_telegraf.yaml @@ -5,7 +5,7 @@ telegraf: advanced: True helpLink: influxdb output: - description: Selects the backend(s) Telegraf writes metrics to. INFLUXDB keeps the current behavior; POSTGRES writes to the grid's Postgres instance; BOTH dual-writes for migration validation. + description: Selects the backend(s) Telegraf writes metrics to. INFLUXDB keeps the current behavior; POSTGRES writes to the grid's Postgres instance; BOTH dual-writes for migration validation. When set to BOTH, the grid screen's metrics are pulled from Postgres, and the InfluxDB tool link remains visible. When set to POSTGRES, the InfluxDB tool link is removed. options: - INFLUXDB - POSTGRES From 7d17784e96857e6ea6620364231eba2dcdff4751 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Fri, 10 Jul 2026 12:50:21 -0500 Subject: [PATCH 24/28] status messages for so-elastic-fleet-setup --- salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup | 5 ++++- setup/so-setup | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup index 2b677606a..0d7b99e48 100755 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup @@ -250,6 +250,7 @@ if ! salt-call state.apply elasticfleet queue=True; then exit 1 fi +printf "\nRunning so-elastic-agent-gen-installers\n" # Generate installers & install Elastic Agent on the node for agent_gen_attempt in {1..3}; do if so-elastic-agent-gen-installers; then @@ -267,4 +268,6 @@ printf "\nApplying elasticfleet.install_agent_grid state\n" if ! salt-call state.apply elasticfleet.install_agent_grid queue=True; then printf "\nFailure(s) in elasticfleet.install_agent_grid state... Exiting...\n" exit 1 -fi \ No newline at end of file +fi + +printf "\nElastic Fleet setup completed successfully\n" \ No newline at end of file diff --git a/setup/so-setup b/setup/so-setup index a44934088..896505ba5 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -793,7 +793,7 @@ if ! [[ -f $install_opt_file ]]; then logCmd "so-soc-restart" title "Setting up Elastic Fleet" logCmd "salt-call state.apply elasticfleet.config" - if ! logCmd so-elastic-fleet-setup; then + if ! so-elastic-fleet-setup; then fail_setup "Failed to run so-elastic-fleet-setup" fi mark_setup_complete From 99e9fc1c3b6be005ebbb31bc053393a61271a240 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Fri, 10 Jul 2026 13:50:55 -0500 Subject: [PATCH 25/28] ES 9.3.7 --- salt/elasticfleet/defaults.yaml | 1 - salt/elasticsearch/defaults.yaml | 2 +- salt/kibana/defaults.yaml | 2 +- salt/manager/tools/sbin/soup | 9 ++++++--- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/salt/elasticfleet/defaults.yaml b/salt/elasticfleet/defaults.yaml index 022600083..a3132d3f4 100644 --- a/salt/elasticfleet/defaults.yaml +++ b/salt/elasticfleet/defaults.yaml @@ -1,6 +1,5 @@ elasticfleet: enabled: False - patch_version: 9.3.3+build202604082258 # Elastic Agent specific patch release. enable_manager_output: True config: server: diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index f0b01b3ca..14d753c21 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -1,6 +1,6 @@ elasticsearch: enabled: false - version: 9.3.3 + version: 9.3.7 index_clean: true data_retention_method: DLM vm: diff --git a/salt/kibana/defaults.yaml b/salt/kibana/defaults.yaml index ecf56756b..2cf6fe92f 100644 --- a/salt/kibana/defaults.yaml +++ b/salt/kibana/defaults.yaml @@ -22,7 +22,7 @@ kibana: - default - file migrations: - discardCorruptObjects: "9.3.3" + discardCorruptObjects: "9.3.7" telemetry: enabled: False xpack: diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index e33c27f6f..79ea3967b 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -740,7 +740,6 @@ fix_logstash_0013_lumberjack_pipeline_name() { up_to_3.1.0() { ensure_postgres_local_pillar ensure_postgres_secret - determine_elastic_agent_upgrade elasticsearch_backup_index_templates # Clear existing component template state file. rm -f /opt/so/state/esfleet_component_templates.json @@ -888,6 +887,9 @@ update_kafka_metadata() { } up_to_3.2.0() { + # download 9.3.7 elastic agent packages + determine_elastic_agent_upgrade + fix_logstash_0013_lumberjack_pipeline_name pin_elasticsearch_data_retention_method @@ -901,7 +903,7 @@ post_to_3.2.0() { bootstrap_so_soc_database - # Including agent regen script here since it was missed in post_to_3.1.0 + # Generate 9.3.7 elastic agent installers echo "Regenerating Elastic Agent Installers" /sbin/so-elastic-agent-gen-installers @@ -1174,7 +1176,8 @@ verify_es_version_compatibility() { ["8.18.4"]="8.18.6 8.18.8 9.0.8" ["8.18.6"]="8.18.8 9.0.8" ["8.18.8"]="9.0.8" - ["9.0.8"]="9.3.3" + ["9.0.8"]="9.3.3 9.3.7" + ) # Elasticsearch MUST upgrade through these versions From 87a56396437f6aacc2a97251de0410fceeca9d07 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 13 Jul 2026 13:05:25 -0500 Subject: [PATCH 26/28] pipeline updates --- .../elastic-defend-endpoints.json | 2 +- .../grid-nodes_general/import-evtx-logs.json | 2 +- ...nse.log-1.25.2 => logs-pfsense.log-1.25.4} | 20 +++++++++---------- ...icata => logs-pfsense.log-1.25.4-suricata} | 0 4 files changed, 12 insertions(+), 12 deletions(-) rename salt/elasticsearch/files/ingest/{logs-pfsense.log-1.25.2 => logs-pfsense.log-1.25.4} (96%) rename salt/elasticsearch/files/ingest/{logs-pfsense.log-1.25.2-suricata => logs-pfsense.log-1.25.4-suricata} (100%) diff --git a/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json b/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json index c27da26f7..7d64fd1ab 100644 --- a/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json +++ b/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json @@ -5,7 +5,7 @@ "package": { "name": "endpoint", "title": "Elastic Defend", - "version": "9.3.0", + "version": "9.3.1", "requires_root": true }, "enabled": true, diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json index 32d210172..be05d59e0 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json @@ -29,7 +29,7 @@ "\\.gz$" ], "include_files": [], - "processors": "- dissect:\n tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"\n- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n- drop_fields:\n fields: [\"host\"]\n ignore_missing: true\n- add_fields:\n target: data_stream\n fields:\n type: logs\n dataset: system.security\n- add_fields:\n target: event\n fields:\n dataset: system.security\n module: system\n imported: true\n- add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.security-2.15.0\n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.sysmon_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.sysmon_operational\n module: windows\n imported: true\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.sysmon_operational-3.8.0\n- if:\n equals:\n winlog.channel: 'Application'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.application\n - add_fields:\n target: event\n fields:\n dataset: system.application\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.application-2.15.0\n- if:\n equals:\n winlog.channel: 'System'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.system\n - add_fields:\n target: event\n fields:\n dataset: system.system\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.system-2.15.0\n \n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.powershell_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.powershell_operational\n module: windows\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.powershell_operational-3.8.0\n- add_fields:\n target: data_stream\n fields:\n dataset: import", + "processors": "- dissect:\n tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"\n- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n- drop_fields:\n fields: [\"host\"]\n ignore_missing: true\n- add_fields:\n target: data_stream\n fields:\n type: logs\n dataset: system.security\n- add_fields:\n target: event\n fields:\n dataset: system.security\n module: system\n imported: true\n- add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.security-2.20.0\n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.sysmon_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.sysmon_operational\n module: windows\n imported: true\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.sysmon_operational-3.8.3\n- if:\n equals:\n winlog.channel: 'Application'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.application\n - add_fields:\n target: event\n fields:\n dataset: system.application\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.application-2.20.0\n- if:\n equals:\n winlog.channel: 'System'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.system\n - add_fields:\n target: event\n fields:\n dataset: system.system\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.system-2.20.0\n \n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.powershell_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.powershell_operational\n module: windows\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.powershell_operational-3.8.3\n- add_fields:\n target: data_stream\n fields:\n dataset: import", "tags": [ "import" ], diff --git a/salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.2 b/salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.4 similarity index 96% rename from salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.2 rename to salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.4 index 1ea828514..c01383fde 100644 --- a/salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.2 +++ b/salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.4 @@ -118,70 +118,70 @@ { "pipeline": { "tag": "pipeline_e16851a7", - "name": "logs-pfsense.log-1.25.2-firewall", + "name": "logs-pfsense.log-1.25.4-firewall", "if": "ctx.event.provider == 'filterlog'" } }, { "pipeline": { "tag": "pipeline_828590b5", - "name": "logs-pfsense.log-1.25.2-openvpn", + "name": "logs-pfsense.log-1.25.4-openvpn", "if": "ctx.event.provider == 'openvpn'" } }, { "pipeline": { "tag": "pipeline_9d37039c", - "name": "logs-pfsense.log-1.25.2-ipsec", + "name": "logs-pfsense.log-1.25.4-ipsec", "if": "ctx.event.provider == 'charon'" } }, { "pipeline": { "tag": "pipeline_ad56bbca", - "name": "logs-pfsense.log-1.25.2-dhcp", + "name": "logs-pfsense.log-1.25.4-dhcp", "if": "[\"dhcpd\", \"dhclient\", \"dhcp6c\", \"dnsmasq-dhcp\"].contains(ctx.event.provider)" } }, { "pipeline": { "tag": "pipeline_dd85553d", - "name": "logs-pfsense.log-1.25.2-unbound", + "name": "logs-pfsense.log-1.25.4-unbound", "if": "ctx.event.provider == 'unbound'" } }, { "pipeline": { "tag": "pipeline_720ed255", - "name": "logs-pfsense.log-1.25.2-haproxy", + "name": "logs-pfsense.log-1.25.4-haproxy", "if": "ctx.event.provider == 'haproxy'" } }, { "pipeline": { "tag": "pipeline_456beba5", - "name": "logs-pfsense.log-1.25.2-php-fpm", + "name": "logs-pfsense.log-1.25.4-php-fpm", "if": "ctx.event.provider == 'php-fpm'" } }, { "pipeline": { "tag": "pipeline_a0d89375", - "name": "logs-pfsense.log-1.25.2-squid", + "name": "logs-pfsense.log-1.25.4-squid", "if": "ctx.event.provider == 'squid'" } }, { "pipeline": { "tag": "pipeline_c2f1ed55", - "name": "logs-pfsense.log-1.25.2-snort", + "name": "logs-pfsense.log-1.25.4-snort", "if": "ctx.event.provider == 'snort'" } }, { "pipeline": { "tag":"pipeline_33db1c9e", - "name": "logs-pfsense.log-1.25.2-suricata", + "name": "logs-pfsense.log-1.25.4-suricata", "if": "ctx.event.provider == 'suricata'" } }, diff --git a/salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.2-suricata b/salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.4-suricata similarity index 100% rename from salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.2-suricata rename to salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.4-suricata From 2cd889782db10210f463bc6aa30586a0f7b83bf9 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 13 Jul 2026 13:16:12 -0500 Subject: [PATCH 27/28] soup es check for 9.3.7 --- salt/manager/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 79ea3967b..a87dfecfc 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -1177,7 +1177,7 @@ verify_es_version_compatibility() { ["8.18.6"]="8.18.8 9.0.8" ["8.18.8"]="9.0.8" ["9.0.8"]="9.3.3 9.3.7" - + ["9.3.3"]="9.3.7" ) # Elasticsearch MUST upgrade through these versions From 63d4061500cd7c4576d64a1b0c0d5cf837d92cb4 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 14 Jul 2026 13:52:32 -0400 Subject: [PATCH 28/28] prevent login redirect to any API url --- salt/nginx/etc/nginx.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/nginx/etc/nginx.conf b/salt/nginx/etc/nginx.conf index b7a70da2b..3f74c411c 100644 --- a/salt/nginx/etc/nginx.conf +++ b/salt/nginx/etc/nginx.conf @@ -399,7 +399,7 @@ http { error_page 429 = @error429; location @error401 { - if ($request_uri ~* (^/api/.*|^/connect/.*|^/oauth2/.*|^/.*\.map$)) { + if ($request_uri ~* (^.*/api/.*|^/connect/.*|^/oauth2/.*|^/.*\.map$)) { return 401; }