From 57b7d59387269ab6a26c45bf7fc5e35cdad57d26 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Tue, 7 Jul 2026 16:42:47 -0500 Subject: [PATCH 1/5] verify elastic-agent reports healthy status before completing installation --- salt/elasticfleet/install_agent_grid.sls | 8 +- .../tools/sbin/so-elastic-agent-install | 75 +++++++++++++++++++ 2 files changed, 78 insertions(+), 5 deletions(-) create mode 100644 salt/elasticfleet/tools/sbin/so-elastic-agent-install diff --git a/salt/elasticfleet/install_agent_grid.sls b/salt/elasticfleet/install_agent_grid.sls index 482af2e1e..431aa6f97 100644 --- a/salt/elasticfleet/install_agent_grid.sls +++ b/salt/elasticfleet/install_agent_grid.sls @@ -21,11 +21,9 @@ pull_agent_installer: run_installer: cmd.run: - - name: ./so-elastic-agent_linux_amd64 -token={{ GRIDNODETOKEN }} -force - - cwd: /opt/so - - retry: - attempts: 3 - interval: 20 + - name: /usr/sbin/so-elastic-agent-install "{{ GRIDNODETOKEN }}" + - require: + - file: pull_agent_installer cleanup_agent_installer: file.absent: diff --git a/salt/elasticfleet/tools/sbin/so-elastic-agent-install b/salt/elasticfleet/tools/sbin/so-elastic-agent-install new file mode 100644 index 000000000..dda3f1672 --- /dev/null +++ b/salt/elasticfleet/tools/sbin/so-elastic-agent-install @@ -0,0 +1,75 @@ +#!/bin/bash +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +. /usr/sbin/so-elastic-fleet-common + + +# passed in as arg from elasticfleet/install_agent_grid.sls, else pulled from pillar later +GRIDNODETOKEN="$1" + +check_agent_health() { + timeout=180 + interval=10 + start=$SECONDS + + while (( SECONDS - start < timeout )); do + if elastic-agent status 2>/dev/null | grep -A1 'elastic-agent$' | grep -q 'status: (HEALTHY)'; then + return 0 + fi + echo "elastic-agent is not yet healthy. Waiting for ${interval} seconds before checking again..." + sleep "$interval" + done + + echo "elastic-agent did not become healthy within ${timeout} seconds" + return 1 +} + +if command -v elastic-agent >/dev/null 2>&1; then + elastic-agent uninstall -f +fi + +if [[ -z "$GRIDNODETOKEN" ]]; then + noderole=$(so-yaml.py get -r /etc/salt/grains role) + if [[ "$noderole" == "so-heavynode" ]]; then + GRIDNODETOKEN=$(salt-call pillar.get global:fleet_grid_enrollment_token_heavy --out=newline_values_only) + else + GRIDNODETOKEN=$(salt-call pillar.get global:fleet_grid_enrollment_token_general --out=newline_values_only) + fi +fi + +if [[ ! -x /opt/so/so-elastic-agent_linux_amd64 ]]; then + echo "Downloading elastic-agent installer... This could take a while if another Salt job is running." + salt-call state.sls_id pull_agent_installer elasticfleet.install_agent_grid queue=True +fi + +if [[ -x /opt/so/so-elastic-agent_linux_amd64 ]]; then + attempts=0 + cd /opt/so/ || exit 1 + + while [[ $attempts -lt 3 ]]; do + if ./so-elastic-agent_linux_amd64 -token="$GRIDNODETOKEN" -force; echo "Verifying elastic-agent health..." && check_agent_health; then + rm -f /opt/so/so-elastic-agent_linux_amd64 + elastic-agent status + + exit 0 + fi + + attempts=$((attempts + 1)) + + if [[ $attempts -lt 3 ]]; then + echo "so-elastic-agent installer failed. Retrying in 20 seconds..." + sleep 20 + fi + done + + echo "so-elastic-agent installer failed after 3 attempts. Exiting." + + exit 1 +else + echo "Unable to locate so-elastic-agent installer. Exiting." + + exit 1 +fi From 70af3cec534dfa2c5122893ce73ed1c9b29632d4 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Tue, 7 Jul 2026 16:48:10 -0500 Subject: [PATCH 2/5] avoid using 'failure' until all loops are done so so-verify doesn't flag it --- .../elasticfleet/tools/sbin/so-elastic-agent-install | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/salt/elasticfleet/tools/sbin/so-elastic-agent-install b/salt/elasticfleet/tools/sbin/so-elastic-agent-install index dda3f1672..fd6bceeff 100644 --- a/salt/elasticfleet/tools/sbin/so-elastic-agent-install +++ b/salt/elasticfleet/tools/sbin/so-elastic-agent-install @@ -19,11 +19,11 @@ check_agent_health() { if elastic-agent status 2>/dev/null | grep -A1 'elastic-agent$' | grep -q 'status: (HEALTHY)'; then return 0 fi - echo "elastic-agent is not yet healthy. Waiting for ${interval} seconds before checking again..." + echo "The Elastic Agent is not yet healthy. Waiting for ${interval} seconds before checking again..." sleep "$interval" done - echo "elastic-agent did not become healthy within ${timeout} seconds" + echo "The Elastic Agent did not become healthy within ${timeout} seconds" return 1 } @@ -41,7 +41,7 @@ if [[ -z "$GRIDNODETOKEN" ]]; then fi if [[ ! -x /opt/so/so-elastic-agent_linux_amd64 ]]; then - echo "Downloading elastic-agent installer... This could take a while if another Salt job is running." + echo "Downloading so-elastic-agent installer... This could take a while if another Salt job is running." salt-call state.sls_id pull_agent_installer elasticfleet.install_agent_grid queue=True fi @@ -50,7 +50,7 @@ if [[ -x /opt/so/so-elastic-agent_linux_amd64 ]]; then cd /opt/so/ || exit 1 while [[ $attempts -lt 3 ]]; do - if ./so-elastic-agent_linux_amd64 -token="$GRIDNODETOKEN" -force; echo "Verifying elastic-agent health..." && check_agent_health; then + if ./so-elastic-agent_linux_amd64 -token="$GRIDNODETOKEN" -force; echo "Verifying Elastic Agent health..." && check_agent_health; then rm -f /opt/so/so-elastic-agent_linux_amd64 elastic-agent status @@ -60,12 +60,12 @@ if [[ -x /opt/so/so-elastic-agent_linux_amd64 ]]; then attempts=$((attempts + 1)) if [[ $attempts -lt 3 ]]; then - echo "so-elastic-agent installer failed. Retrying in 20 seconds..." + echo "Unable to verify Elastic Agent health... Retrying in 20 seconds..." sleep 20 fi done - echo "so-elastic-agent installer failed after 3 attempts. Exiting." + echo "The so-elastic-agent installer failed after 3 attempts. Exiting." exit 1 else From f8de176f4bbbde8b8fd5f6aae06b48c93d1709b1 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Wed, 8 Jul 2026 07:47:47 -0500 Subject: [PATCH 3/5] uninstall agent on final failed attempt --- .../tools/sbin/so-elastic-agent-install | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/salt/elasticfleet/tools/sbin/so-elastic-agent-install b/salt/elasticfleet/tools/sbin/so-elastic-agent-install index fd6bceeff..920b17137 100644 --- a/salt/elasticfleet/tools/sbin/so-elastic-agent-install +++ b/salt/elasticfleet/tools/sbin/so-elastic-agent-install @@ -27,9 +27,13 @@ check_agent_health() { return 1 } -if command -v elastic-agent >/dev/null 2>&1; then - elastic-agent uninstall -f -fi +uninstall_agent() { + if command -v elastic-agent >/dev/null 2>&1; then + elastic-agent uninstall -f + fi +} + +uninstall_agent if [[ -z "$GRIDNODETOKEN" ]]; then noderole=$(so-yaml.py get -r /etc/salt/grains role) @@ -40,6 +44,12 @@ if [[ -z "$GRIDNODETOKEN" ]]; then fi fi +if [[ -z "$GRIDNODETOKEN" ]]; then + echo "Unable to determine Elastic Fleet enrollment token. Exiting." + + exit 1 +fi + if [[ ! -x /opt/so/so-elastic-agent_linux_amd64 ]]; then echo "Downloading so-elastic-agent installer... This could take a while if another Salt job is running." salt-call state.sls_id pull_agent_installer elasticfleet.install_agent_grid queue=True @@ -50,7 +60,7 @@ if [[ -x /opt/so/so-elastic-agent_linux_amd64 ]]; then cd /opt/so/ || exit 1 while [[ $attempts -lt 3 ]]; do - if ./so-elastic-agent_linux_amd64 -token="$GRIDNODETOKEN" -force; echo "Verifying Elastic Agent health..." && check_agent_health; then + if ./so-elastic-agent_linux_amd64 -token="$GRIDNODETOKEN" -force && echo "Verifying Elastic Agent health..." && check_agent_health; then rm -f /opt/so/so-elastic-agent_linux_amd64 elastic-agent status @@ -65,6 +75,8 @@ if [[ -x /opt/so/so-elastic-agent_linux_amd64 ]]; then fi done + uninstall_agent + rm -f /opt/so/so-elastic-agent_linux_amd64 echo "The so-elastic-agent installer failed after 3 attempts. Exiting." exit 1 From 2a4a7307f70d2b39229b3abb545d80554999a51f Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Wed, 8 Jul 2026 07:51:13 -0500 Subject: [PATCH 4/5] uninstall agent after downloading new one and getting gridtoken --- salt/elasticfleet/tools/sbin/so-elastic-agent-install | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/elasticfleet/tools/sbin/so-elastic-agent-install b/salt/elasticfleet/tools/sbin/so-elastic-agent-install index 920b17137..bda8306f0 100644 --- a/salt/elasticfleet/tools/sbin/so-elastic-agent-install +++ b/salt/elasticfleet/tools/sbin/so-elastic-agent-install @@ -33,8 +33,6 @@ uninstall_agent() { fi } -uninstall_agent - if [[ -z "$GRIDNODETOKEN" ]]; then noderole=$(so-yaml.py get -r /etc/salt/grains role) if [[ "$noderole" == "so-heavynode" ]]; then @@ -55,6 +53,8 @@ if [[ ! -x /opt/so/so-elastic-agent_linux_amd64 ]]; then salt-call state.sls_id pull_agent_installer elasticfleet.install_agent_grid queue=True fi +uninstall_agent + if [[ -x /opt/so/so-elastic-agent_linux_amd64 ]]; then attempts=0 cd /opt/so/ || exit 1 From 85d7f6bebcf406fde9425171b12a1b5f182ac076 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Wed, 8 Jul 2026 08:19:15 -0500 Subject: [PATCH 5/5] independently download so-elastic-agent installer --- .../tools/sbin/so-elastic-agent-install | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/salt/elasticfleet/tools/sbin/so-elastic-agent-install b/salt/elasticfleet/tools/sbin/so-elastic-agent-install index bda8306f0..c6a499e94 100644 --- a/salt/elasticfleet/tools/sbin/so-elastic-agent-install +++ b/salt/elasticfleet/tools/sbin/so-elastic-agent-install @@ -50,15 +50,23 @@ fi if [[ ! -x /opt/so/so-elastic-agent_linux_amd64 ]]; then echo "Downloading so-elastic-agent installer... This could take a while if another Salt job is running." - salt-call state.sls_id pull_agent_installer elasticfleet.install_agent_grid queue=True -fi -uninstall_agent + # When running outside of elasticfleet/install_agent_grid.sls we need to download the installer independently. + # PYTHONWARNINGS="ignore" to avoid messages like the following when running salt-call: + # '/opt/saltstack/salt/lib/python3.10/site-packages/salt/transport/base.py:129: TransportWarning: Unclosed transport! + # File "/bin/salt-call", line 12, in + # sys.exit(salt_call())' + + PYTHONWARNINGS="ignore" salt-call state.single file.managed name=/opt/so/so-elastic-agent_linux_amd64 source=salt://elasticfleet/files/so_agent-installers/so-elastic-agent_linux_amd64 mode=755 makedirs=True queue=True + +fi if [[ -x /opt/so/so-elastic-agent_linux_amd64 ]]; then attempts=0 cd /opt/so/ || exit 1 + uninstall_agent + while [[ $attempts -lt 3 ]]; do if ./so-elastic-agent_linux_amd64 -token="$GRIDNODETOKEN" -force && echo "Verifying Elastic Agent health..." && check_agent_health; then rm -f /opt/so/so-elastic-agent_linux_amd64