soup: make failed upgrades and hotfixes resumable

A failed highstate mid-upgrade left /etc/soversion already advanced to the
target version (the highstate stamps it from the pillar via the soversionfile
state), so a re-run of soup saw INSTALLEDVERSION == NEWVERSION and reported
"already running the latest version", stranding the box with post-upgrade
steps never run.

Introduce /etc/sopostversion, a soup-owned marker (no salt state manages it)
that records post-upgrade walk progress. It is seeded from the pre-upgrade
version before the highstate, advanced after each post_to_* step, and removed
on successful completion. upgrade_check treats a leftover marker as "upgrade
not finished" and resumes the remaining post steps instead of bailing.

Also fix the hotfix path: /etc/sohotfix was written before the hotfix
highstate, so a failed hotfix highstate looked already-applied on re-run.
Since no salt state manages /etc/sohotfix, defer its write (update_version)
until after the highstate succeeds so it is an honest completion marker.

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup

@@ -257,6 +257,7 @@ for state_attempt in {1..3}; do
     fi
 done
 
+printf "\nRunning so-elastic-agent-gen-installers\n"
 # Generate installers & install Elastic Agent on the node
 for agent_gen_attempt in {1..3}; do
     if so-elastic-agent-gen-installers; then
@@ -274,4 +275,6 @@ printf "\nApplying elasticfleet.install_agent_grid state\n"
 if ! salt-call state.apply elasticfleet.install_agent_grid queue=True; then
     printf "\nFailure(s) in elasticfleet.install_agent_grid state... Exiting...\n"
     exit 1
-fi
+fi
+
+printf "\nElastic Fleet setup completed successfully\n"

salt/manager/tools/sbin/soup

@@ -12,7 +12,17 @@
 UPDATE_DIR=/tmp/sogh/securityonion
 DEFAULT_SALT_DIR=/opt/so/saltstack/default
 INSTALLEDVERSION=$(cat /etc/soversion)
-POSTVERSION=$INSTALLEDVERSION
+# /etc/sopostversion is a soup-owned marker (no salt state manages it) tracking how
+# far the post-upgrade walk has progressed. Its presence means a prior upgrade did
+# not finish its post-upgrade steps; its contents are the resume point. It is read
+# here before preupgrade_changes mutates INSTALLEDVERSION and before any highstate
+# stamps /etc/soversion from the pillar.
+POSTVERSION_FILE=/etc/sopostversion
+if [ -f "$POSTVERSION_FILE" ]; then
+  POSTVERSION=$(cat "$POSTVERSION_FILE")
+else
+  POSTVERSION=$INSTALLEDVERSION
+fi
 INSTALLEDSALTVERSION=$(salt --versions-report | grep Salt: | awk '{print $2}')
 BATCHSIZE=5
 SOUP_LOG=/root/soup.log
@@ -414,6 +424,13 @@ preupgrade_changes() {
     true
 }
 
+set_postversion() {
+    # Persist post-upgrade walk progress so an interrupted upgrade can resume the
+    # remaining steps on the next soup run (see /etc/sopostversion handling).
+    POSTVERSION="$1"
+    echo "$POSTVERSION" > "$POSTVERSION_FILE"
+}
+
 postupgrade_changes() {
     # This function is to add any new pillar items if needed.
     echo "Running post upgrade processes."
@@ -421,6 +438,8 @@ postupgrade_changes() {
     [[ "$POSTVERSION" =~ ^2\.4\.21[0-9]+$ ]] && post_to_3.0.0
     [[ "$POSTVERSION" == "3.0.0" ]] && post_to_3.1.0
     [[ "$POSTVERSION" == "3.1.0" ]] && post_to_3.2.0
+    # All applicable post-upgrade steps completed; clear the resume marker.
+    rm -f "$POSTVERSION_FILE"
     true
 }
 
@@ -513,7 +532,7 @@ post_to_3.0.0() {
   # convert yes/no in suricata pillars to true/false
   convert_suricata_yes_no
 
-  POSTVERSION=3.0.0
+  set_postversion 3.0.0
 }
 
 ### 3.0.0 End ###
@@ -776,7 +795,7 @@ post_to_3.1.0() {
   # Check for unhealthy / unauthorized integration transform jobs and attempt reauthorizations
   check_transform_health_and_reauthorize || true
 
-  POSTVERSION=3.1.0
+  set_postversion 3.1.0
 }
 
 ### 3.1.0 End ###
@@ -911,7 +930,7 @@ post_to_3.2.0() {
 
   update_kafka_metadata "4.3"
 
-  POSTVERSION=3.2.0
+  set_postversion 3.2.0
 }
 
 ### 3.2.0 End ###
@@ -1063,6 +1082,14 @@ upgrade_check() {
   fi
   [[ -f /etc/sohotfix ]] && CURRENTHOTFIX=$(cat /etc/sohotfix)
   if [ "$INSTALLEDVERSION" == "$NEWVERSION" ]; then
+    # A leftover post-version marker means a previous upgrade to this version
+    # advanced /etc/soversion (the highstate stamps it from the pillar) but did not
+    # finish its post-upgrade steps. Resume the upgrade instead of reporting "latest".
+    if [ -f "$POSTVERSION_FILE" ] && [ "$(cat "$POSTVERSION_FILE")" != "$NEWVERSION" ]; then
+      echo "A previous upgrade to $NEWVERSION did not complete its post-upgrade steps; resuming."
+      is_hotfix=false
+      return 0
+    fi
     echo "Checking to see if there are hotfixes needed"
     if [ "$HOTFIXVERSION" == "$CURRENTHOTFIX" ]; then
       echo "You are already running the latest version of Security Onion."
@@ -1814,9 +1841,14 @@ main() {
     create_local_directories "/opt/so/saltstack/default"
     apply_hotfix
     echo "Hotfix applied"
-    update_version
     enable_highstate
     highstate
+    # Record the hotfix only after the highstate succeeds. /etc/sohotfix is written
+    # solely by soup (no salt state manages it), so deferring the write means a failed
+    # hotfix highstate leaves the old hotfix value and re-running soup re-applies it,
+    # rather than reporting "already latest". The soversion/pillar writes in
+    # update_version are no-ops here since the version is unchanged for a hotfix.
+    update_version
   else
     echo ""
     echo "Performing upgrade from Security Onion $INSTALLEDVERSION to Security Onion $NEWVERSION."
@@ -1873,6 +1905,10 @@ main() {
     copy_new_files
     echo ""
     create_local_directories "/opt/so/saltstack/default"
+    # Seed the resume marker before the highstate stamps /etc/soversion to the new
+    # version, so an interrupted upgrade is detectable as "not finished" on re-run.
+    # POSTVERSION still holds the pre-upgrade (or prior resume) version here.
+    [ -f "$POSTVERSION_FILE" ] || echo "$POSTVERSION" > "$POSTVERSION_FILE"
     update_version
 
     echo ""

salt/nginx/etc/nginx.conf

@@ -399,7 +399,7 @@ http {
 		error_page 429 = @error429;
 
 		location @error401 {
-			if ($request_uri ~* (^/api/.*|^/connect/.*|^/oauth2/.*|^/.*\.map$)) {
+			if ($request_uri ~* (^.*/api/.*|^/connect/.*|^/oauth2/.*|^/.*\.map$)) {
 				return    401;
 			}
 

salt/soc/defaults.map.jinja

@@ -8,6 +8,7 @@
 {% from 'docker/docker.map.jinja' import DOCKERMERGED -%}
 {% set INFLUXDB_TOKEN = salt['pillar.get']('influxdb:token') %}
 {% import_text 'influxdb/metrics_link.txt' as METRICS_LINK %}
+{% from 'telegraf/map.jinja' import TELEGRAFMERGED %}
 
 {% for module, application_url in GLOBALS.application_urls.items() %}
 {%   do SOCDEFAULTS.soc.config.server.modules[module].update({'hostUrl': application_url}) %}
@@ -24,13 +25,22 @@
 
 {% do SOCDEFAULTS.soc.config.server.modules.elastic.update({'username': GLOBALS.elasticsearch.auth.users.so_elastic_user.user, 'password': GLOBALS.elasticsearch.auth.users.so_elastic_user.pass}) %}
 
-{% do SOCDEFAULTS.soc.config.server.modules.influxdb.update({'hostUrl': 'https://' ~ GLOBALS.influxdb_host ~ ':8086'}) %}
-{% do SOCDEFAULTS.soc.config.server.modules.influxdb.update({'token': INFLUXDB_TOKEN}) %}
-{% for tool in SOCDEFAULTS.soc.config.server.client.tools %}
-{%   if tool.name == "toolInfluxDb" and METRICS_LINK | length > 0 %}
-{%     do tool.update({'link': METRICS_LINK}) %}
-{%   endif %}
-{% endfor %}
+{% if TELEGRAFMERGED.output == 'POSTGRES' %}
+{%   for tool in SOCDEFAULTS.soc.config.server.client.tools %}
+{%     if tool.name == "toolInfluxDb" %}
+{%       do SOCDEFAULTS.soc.config.server.client.tools.remove(tool) %}
+{%     endif %}
+{%   endfor %}
+
+{% else %}
+{%   do SOCDEFAULTS.soc.config.server.modules.influxdb.update({'hostUrl': 'https://' ~ GLOBALS.influxdb_host ~ ':8086'}) %}
+{%   do SOCDEFAULTS.soc.config.server.modules.influxdb.update({'token': INFLUXDB_TOKEN}) %}
+{%   for tool in SOCDEFAULTS.soc.config.server.client.tools %}
+{%     if tool.name == "toolInfluxDb" and METRICS_LINK | length > 0 %}
+{%       do tool.update({'link': METRICS_LINK}) %}
+{%     endif %}
+{%   endfor %}
+{% endif %}
 
 {% do SOCDEFAULTS.soc.config.server.modules.statickeyauth.update({'anonymousCidr': DOCKERMERGED.range, 'apiKey': pillar.sensoroni.config.sensoronikey}) %}
 

salt/soc/merged.map.jinja

@@ -7,6 +7,11 @@
 {% from 'soc/defaults.map.jinja' import SOCDEFAULTS with context %}
 {% from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_NODES %}
 {% from 'manager/map.jinja' import MANAGERMERGED %}
+{% from 'telegraf/map.jinja' import TELEGRAFMERGED %}
+{%- set PG_ENTRY = salt['pillar.get']('telegraf:postgres_creds:' ~ grains.id, {}) %}
+{%- set PG_USER = PG_ENTRY.get('user', '') %}
+{%- set PG_PASS = PG_ENTRY.get('pass', '') %}
+
 {% set DOCKER_EXTRA_HOSTS = ELASTICSEARCH_NODES %}
 {% do DOCKER_EXTRA_HOSTS.append({GLOBALS.influxdb_host:pillar.node_data[GLOBALS.influxdb_host].ip}) %}
 
@@ -75,6 +80,20 @@
 {%   do SOCMERGED.config.server.update({'airgapEnabled': false}) %}
 {% endif %}
 
+{# Define the postgresmetrics module if telegraf is setup to only use Postgres #}
+{% if TELEGRAFMERGED.output != 'INFLUXDB' and PG_USER and PG_PASS %}
+{%   do SOCMERGED.config.server.modules.update({
+        'postgresmetrics': {
+          'database': 'so_telegraf',
+          'host': GLOBALS.manager_ip,
+          'password': PG_PASS,
+          'port': 5432,
+          'sslMode': 'allow',
+          'user': PG_USER,
+        }
+      }) %}
+{%   do SOCMERGED.config.server.modules.pop('influxdb') %}
+{% endif %}
 
 {# Define the Detections custom ruleset that should always be present #}
 {% set CUSTOM_RULESET = {

salt/telegraf/soc_telegraf.yaml

@@ -5,7 +5,7 @@ telegraf:
     advanced: True
     helpLink: influxdb
   output:
-    description: Selects the backend(s) Telegraf writes metrics to. INFLUXDB keeps the current behavior; POSTGRES writes to the grid's Postgres instance; BOTH dual-writes for migration validation.
+    description: Selects the backend(s) Telegraf writes metrics to. INFLUXDB keeps the current behavior; POSTGRES writes to the grid's Postgres instance; BOTH dual-writes for migration validation. When set to BOTH, the grid screen's metrics are pulled from Postgres, and the InfluxDB tool link remains visible. When set to POSTGRES, the InfluxDB tool link is removed.
     options:
       - INFLUXDB
       - POSTGRES

setup/so-setup

@@ -793,7 +793,7 @@ if ! [[ -f $install_opt_file ]]; then
 		logCmd "so-soc-restart"
 		title "Setting up Elastic Fleet"
 		logCmd "salt-call state.apply elasticfleet.config"	
-		if ! logCmd so-elastic-fleet-setup; then
+		if ! so-elastic-fleet-setup; then
 			fail_setup "Failed to run so-elastic-fleet-setup"
 		fi
 		mark_setup_complete