clear stale docker network endpoints before starting containers

docker_container.running fails with an address-already-in-use error when a
container holds a stale sobridge endpoint - either after its IP changes in
pillar or after an unclean shutdown leaves the endpoint behind.

Add the clear_stale_endpoint macro and call it from every enabled.sls whose
container attaches to sobridge with a static IP. The macro force-disconnects
the container from the network only when its current endpoint IP does not
match the one salt is about to assign, so it is a no-op on a normal highstate.

The strelka states use state IDs that differ from their container names, so
those calls pass state_id explicitly. nginx resolves its IP through
container_config, which varies by role.

salt/docker/macros/docker_endpoint.jinja

@@ -0,0 +1,12 @@
+{% macro clear_stale_endpoint(container, network, ipv4, state_id=None) %}
+{{ container }}_{{ network }}_stale_endpoint:
+  cmd.run:
+    - name: docker network disconnect -f {{ network }} {{ container }} || true
+    - onlyif: docker inspect {{ container }}
+    - unless: >-
+        docker inspect -f
+        '{{ '{{' }} with index .NetworkSettings.Networks "{{ network }}" {{ '}}' }}{{ '{{' }} .IPAMConfig.IPv4Address {{ '}}' }}{{ '{{' }} end {{ '}}' }}'
+        {{ container }} 2>/dev/null | grep -qx '{{ ipv4 }}'
+    - require_in:
+      - docker_container: {{ state_id or container }}
+{% endmacro %}

salt/elastalert/enabled.sls

@@ -7,6 +7,9 @@
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
+{%   from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
+
+{{ clear_stale_endpoint('so-elastalert', 'sobridge', DOCKERMERGED.containers['so-elastalert'].ip) }}
 
 include:
   - elastalert.config

salt/elastic-fleet-package-registry/enabled.sls

@@ -7,6 +7,9 @@
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
+{%   from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
+
+{{ clear_stale_endpoint('so-elastic-fleet-package-registry', 'sobridge', DOCKERMERGED.containers['so-elastic-fleet-package-registry'].ip) }}
 
 include:
   - elastic-fleet-package-registry.config

salt/elasticagent/enabled.sls

@@ -7,6 +7,9 @@
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
+{%   from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
+
+{{ clear_stale_endpoint('so-elastic-agent', 'sobridge', DOCKERMERGED.containers['so-elastic-agent'].ip) }}
 
 include:
   - ca

salt/elasticfleet/defaults.yaml

@@ -1,5 +1,6 @@
 elasticfleet:
   enabled: False
+  patch_version: 9.3.3+build202604082258 # Elastic Agent specific patch release.
   enable_manager_output: True
   config:
     server:

salt/elasticfleet/enabled.sls

@@ -8,6 +8,7 @@
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {%   from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %}
+{%   from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
 
 {#   This value is generated during node install and stored in minion pillar #}
 {%   set SERVICETOKEN = salt['pillar.get']('elasticfleet:config:server:es_token','') %}
@@ -43,6 +44,8 @@ elasticagent_syncartifacts:
 {% endif %}
 
 {%   if SERVICETOKEN != '' %}
+{{ clear_stale_endpoint('so-elastic-fleet', 'sobridge', DOCKERMERGED.containers['so-elastic-fleet'].ip) }}
+
 so-elastic-fleet:
   docker_container.running:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent:{{ GLOBALS.so_version }}

salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json

@@ -5,7 +5,7 @@
   "package": {
     "name": "endpoint",
     "title": "Elastic Defend",
-    "version": "9.3.1",
+    "version": "9.3.0",
     "requires_root": true
   },
   "enabled": true,

salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json

@@ -29,7 +29,7 @@
               "\\.gz$"
             ],
             "include_files": [],
-            "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- drop_fields:\n    fields: [\"host\"]\n    ignore_missing: true\n- add_fields:\n    target: data_stream\n    fields:\n      type: logs\n      dataset: system.security\n- add_fields:\n    target: event\n    fields:\n      dataset: system.security\n      module: system\n      imported: true\n- add_fields:\n    target: \"@metadata\"\n    fields:\n        pipeline: logs-system.security-2.20.0\n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.sysmon_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.sysmon_operational\n          module: windows\n          imported: true\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.sysmon_operational-3.8.3\n- if:\n    equals:\n      winlog.channel: 'Application'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.application-2.20.0\n- if:\n    equals:\n      winlog.channel: 'System'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.system-2.20.0\n    \n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.powershell_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell_operational\n          module: windows\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.powershell_operational-3.8.3\n- add_fields:\n    target: data_stream\n    fields:\n        dataset: import",
+            "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- drop_fields:\n    fields: [\"host\"]\n    ignore_missing: true\n- add_fields:\n    target: data_stream\n    fields:\n      type: logs\n      dataset: system.security\n- add_fields:\n    target: event\n    fields:\n      dataset: system.security\n      module: system\n      imported: true\n- add_fields:\n    target: \"@metadata\"\n    fields:\n        pipeline: logs-system.security-2.15.0\n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.sysmon_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.sysmon_operational\n          module: windows\n          imported: true\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.sysmon_operational-3.8.0\n- if:\n    equals:\n      winlog.channel: 'Application'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.application-2.15.0\n- if:\n    equals:\n      winlog.channel: 'System'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.system-2.15.0\n    \n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.powershell_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell_operational\n          module: windows\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.powershell_operational-3.8.0\n- add_fields:\n    target: data_stream\n    fields:\n        dataset: import",
             "tags": [
               "import"
             ],

salt/elasticsearch/defaults.yaml

@@ -1,6 +1,6 @@
 elasticsearch:
   enabled: false
-  version: 9.3.7
+  version: 9.3.3
   index_clean: true
   data_retention_method: DLM
   vm:

salt/elasticsearch/enabled.sls

@@ -10,6 +10,9 @@
 {%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_NODES %}
 {%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_SEED_HOSTS %}
 {%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
+{%   from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
+
+{{ clear_stale_endpoint('so-elasticsearch', 'sobridge', DOCKERMERGED.containers['so-elasticsearch'].ip) }}
 
 include:
   - ca

salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.2

@@ -118,70 +118,70 @@
     {
       "pipeline": {
         "tag": "pipeline_e16851a7",
-        "name": "logs-pfsense.log-1.25.4-firewall",
+        "name": "logs-pfsense.log-1.25.2-firewall",
         "if": "ctx.event.provider == 'filterlog'"
       }
     },
     {
       "pipeline": {
         "tag": "pipeline_828590b5",
-        "name": "logs-pfsense.log-1.25.4-openvpn",
+        "name": "logs-pfsense.log-1.25.2-openvpn",
         "if": "ctx.event.provider == 'openvpn'"
       }
     },
     {
       "pipeline": {
         "tag": "pipeline_9d37039c",
-        "name": "logs-pfsense.log-1.25.4-ipsec",
+        "name": "logs-pfsense.log-1.25.2-ipsec",
         "if": "ctx.event.provider == 'charon'"
       }
     },
     {
       "pipeline": {
         "tag": "pipeline_ad56bbca",
-        "name": "logs-pfsense.log-1.25.4-dhcp",
+        "name": "logs-pfsense.log-1.25.2-dhcp",
         "if": "[\"dhcpd\", \"dhclient\", \"dhcp6c\", \"dnsmasq-dhcp\"].contains(ctx.event.provider)"
       }
     },
     {
       "pipeline": {
         "tag": "pipeline_dd85553d",
-        "name": "logs-pfsense.log-1.25.4-unbound",
+        "name": "logs-pfsense.log-1.25.2-unbound",
         "if": "ctx.event.provider == 'unbound'"
       }
     },
     {
       "pipeline": {
         "tag": "pipeline_720ed255",
-        "name": "logs-pfsense.log-1.25.4-haproxy",
+        "name": "logs-pfsense.log-1.25.2-haproxy",
         "if": "ctx.event.provider == 'haproxy'"
       }
     },
     {
       "pipeline": {
         "tag": "pipeline_456beba5",
-        "name": "logs-pfsense.log-1.25.4-php-fpm",
+        "name": "logs-pfsense.log-1.25.2-php-fpm",
         "if": "ctx.event.provider == 'php-fpm'"
       }
     },
     {
       "pipeline": {
         "tag": "pipeline_a0d89375",
-        "name": "logs-pfsense.log-1.25.4-squid",
+        "name": "logs-pfsense.log-1.25.2-squid",
         "if": "ctx.event.provider == 'squid'"
       }
     },
     {
       "pipeline": {
         "tag": "pipeline_c2f1ed55",
-        "name": "logs-pfsense.log-1.25.4-snort",
+        "name": "logs-pfsense.log-1.25.2-snort",
         "if": "ctx.event.provider == 'snort'"
       }
     },
     {
       "pipeline": {
         "tag":"pipeline_33db1c9e",
-        "name": "logs-pfsense.log-1.25.4-suricata",
+        "name": "logs-pfsense.log-1.25.2-suricata",
         "if": "ctx.event.provider == 'suricata'"
       }
     },

salt/hydra/enabled.sls

@@ -14,6 +14,9 @@
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 {%   if 'api' in salt['pillar.get']('features', []) %}
+{%   from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
+
+{{ clear_stale_endpoint('so-hydra', 'sobridge', DOCKERMERGED.containers['so-hydra'].ip) }}
 
 include:
   - hydra.config

salt/influxdb/enabled.sls

@@ -9,6 +9,9 @@
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {%   set PASSWORD = salt['pillar.get']('secrets:influx_pass') %}
 {%   set TOKEN = salt['pillar.get']('influxdb:token') %}
+{%   from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
+
+{{ clear_stale_endpoint('so-influxdb', 'sobridge', DOCKERMERGED.containers['so-influxdb'].ip) }}
 
 include:
   - influxdb.ssl

salt/kafka/enabled.sls

@@ -16,6 +16,9 @@
 {%   set KAFKANODES = salt['pillar.get']('kafka:nodes') %}
 {%   set KAFKA_EXTERNAL_ACCESS = salt['pillar.get']('kafka:config:external_access:enabled', default=False) %}
 {%   if 'gmd' in salt['pillar.get']('features', []) %}
+{%   from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
+
+{{ clear_stale_endpoint('so-kafka', 'sobridge', DOCKERMERGED.containers['so-kafka'].ip) }}
 
 include:
   - kafka.ca

salt/kibana/defaults.yaml

@@ -22,7 +22,7 @@ kibana:
           - default
           - file
     migrations:
-      discardCorruptObjects: "9.3.7"
+      discardCorruptObjects: "9.3.3"
     telemetry:
       enabled: False
     xpack:

salt/kibana/enabled.sls

@@ -8,6 +8,9 @@
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
+
+{{ clear_stale_endpoint('so-kibana', 'sobridge', DOCKERMERGED.containers['so-kibana'].ip) }}
 
 include:
   - kibana.config

salt/kratos/enabled.sls

@@ -7,6 +7,9 @@
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
+
+{{ clear_stale_endpoint('so-kratos', 'sobridge', DOCKERMERGED.containers['so-kratos'].ip) }}
 
 include:
   - kratos.config

salt/logstash/enabled.sls

@@ -10,6 +10,9 @@
 {%   from 'logstash/map.jinja' import LOGSTASH_MERGED %}
 {%   from 'logstash/map.jinja' import LOGSTASH_NODES %}
 {%   set lsheap = LOGSTASH_MERGED.settings.lsheap %}
+{%   from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
+
+{{ clear_stale_endpoint('so-logstash', 'sobridge', DOCKERMERGED.containers['so-logstash'].ip) }}
 
 include:
   - ca

salt/manager/tools/sbin/soup

@@ -12,17 +12,7 @@
 UPDATE_DIR=/tmp/sogh/securityonion
 DEFAULT_SALT_DIR=/opt/so/saltstack/default
 INSTALLEDVERSION=$(cat /etc/soversion)
-# /etc/sopostversion is a soup-owned marker (no salt state manages it) tracking how
+POSTVERSION=$INSTALLEDVERSION
-# far the post-upgrade walk has progressed. Its presence means a prior upgrade did
-# not finish its post-upgrade steps; its contents are the resume point. It is read
-# here before preupgrade_changes mutates INSTALLEDVERSION and before any highstate
-# stamps /etc/soversion from the pillar.
-POSTVERSION_FILE=/etc/sopostversion
-if [ -f "$POSTVERSION_FILE" ]; then
-  POSTVERSION=$(cat "$POSTVERSION_FILE")
-else
-  POSTVERSION=$INSTALLEDVERSION
-fi
 INSTALLEDSALTVERSION=$(salt --versions-report | grep Salt: | awk '{print $2}')
 BATCHSIZE=5
 SOUP_LOG=/root/soup.log
@@ -33,10 +23,6 @@ NOTIFYCUSTOMELASTICCONFIG=false
 TOPFILE=/opt/so/saltstack/default/salt/top.sls
 BACKUPTOPFILE=/opt/so/saltstack/default/salt/top.sls.backup
 SALTUPGRADED=false
-# Set true once soup begins modifying the system (past the pre-flight checks), so the
-# EXIT trap can tell the user the update did not finish and must be re-run. Only the
-# pre-flight gates (ES compatibility, disk, network) fail before this is set.
-SOUP_UPGRADE_STARTED=false
 SALT_CLOUD_INSTALLED=false
 SALT_CLOUD_CONFIGURED=false
 # Check if salt-cloud is installed
@@ -137,28 +123,6 @@ check_err() {
 
     echo "SOUP XTRACE debug log (if enabled) at $SOUP_DEBUG_LOG. Re-run soup with SOUP_DEBUG=1 to create $SOUP_DEBUG_LOG"
 
-    # If soup had already started modifying the system, make it unmistakable that the
-    # update is incomplete and must be re-run. soup is resumable: a version upgrade
-    # picks up from the /etc/sopostversion marker, and a hotfix re-applies because
-    # /etc/sohotfix is only advanced after a successful highstate.
-    if [[ "$SOUP_UPGRADE_STARTED" == "true" ]]; then
-      echo ""
-      echo "=============================================================================="
-      echo " UPGRADE INCOMPLETE"
-      echo "=============================================================================="
-      echo " This soup run did NOT finish. Your Security Onion installation may be in a"
-      echo " partially-updated state and is not yet fully upgraded."
-      echo ""
-      echo " Review the error above and $SOUP_LOG, resolve the underlying problem, then"
-      echo " run soup again to resume and complete the update:"
-      echo ""
-      echo "     sudo soup"
-      echo ""
-      echo " soup is resumable -- re-running it continues from where this run stopped."
-      echo "=============================================================================="
-      echo ""
-    fi
-
     exit $exit_code
   fi
 
@@ -327,30 +291,6 @@ check_pillar_items() {
   fi
 }
 
-check_cluster_health() {
-  echo "Checking Elasticsearch cluster health."
-  # Require a 'green' cluster before upgrading; anything less (yellow, red, or
-  # unreachable) blocks. Modeled on the wait used in so-elasticsearch-roles-load.
-  if so-elasticsearch-query "_cluster/health?wait_for_status=green&timeout=120s" --fail > /dev/null 2>&1; then
-    printf "\nThe Elasticsearch cluster is healthy (green). We can proceed with SOUP.\n\n"
-  else
-    printf "\nThe Elasticsearch cluster is not green. Please resolve the cluster health issue so the cluster is green before running SOUP again.\n\n"
-    exit 0
-  fi
-}
-
-check_fleet_server() {
-  echo "Checking that Elastic Fleet Server is responding."
-  # Modeled on the wait_for_so-elastic-fleet state check in elasticfleet/enabled.sls,
-  # which waits for HTTP 200 from the Fleet Server status API.
-  if curl -sk --fail --retry 3 --retry-delay 10 --max-time 30 "https://localhost:8220/api/status" > /dev/null 2>&1; then
-    printf "\nElastic Fleet Server is responding. We can proceed with SOUP.\n\n"
-  else
-    printf "\nElastic Fleet Server is not responding at https://localhost:8220/api/status. Please ensure Elastic Fleet is healthy before running SOUP again.\n\n"
-    exit 0
-  fi
-}
-
 check_saltmaster_status() {
   set +e
   echo "Waiting on the Salt Master service to be ready."
@@ -474,13 +414,6 @@ preupgrade_changes() {
     true
 }
 
-set_postversion() {
-    # Persist post-upgrade walk progress so an interrupted upgrade can resume the
-    # remaining steps on the next soup run (see /etc/sopostversion handling).
-    POSTVERSION="$1"
-    echo "$POSTVERSION" > "$POSTVERSION_FILE"
-}
-
 postupgrade_changes() {
     # This function is to add any new pillar items if needed.
     echo "Running post upgrade processes."
@@ -488,8 +421,6 @@ postupgrade_changes() {
     [[ "$POSTVERSION" =~ ^2\.4\.21[0-9]+$ ]] && post_to_3.0.0
     [[ "$POSTVERSION" == "3.0.0" ]] && post_to_3.1.0
     [[ "$POSTVERSION" == "3.1.0" ]] && post_to_3.2.0
-    # All applicable post-upgrade steps completed; clear the resume marker.
-    rm -f "$POSTVERSION_FILE"
     true
 }
 
@@ -582,7 +513,7 @@ post_to_3.0.0() {
   # convert yes/no in suricata pillars to true/false
   convert_suricata_yes_no
 
-  set_postversion 3.0.0
+  POSTVERSION=3.0.0
 }
 
 ### 3.0.0 End ###
@@ -809,6 +740,7 @@ fix_logstash_0013_lumberjack_pipeline_name() {
 up_to_3.1.0() {
   ensure_postgres_local_pillar
   ensure_postgres_secret
+  determine_elastic_agent_upgrade
   elasticsearch_backup_index_templates
   # Clear existing component template state file.
   rm -f /opt/so/state/esfleet_component_templates.json
@@ -845,7 +777,7 @@ post_to_3.1.0() {
   # Check for unhealthy / unauthorized integration transform jobs and attempt reauthorizations
   check_transform_health_and_reauthorize || true
 
-  set_postversion 3.1.0
+  POSTVERSION=3.1.0
 }
 
 ### 3.1.0 End ###
@@ -956,9 +888,6 @@ update_kafka_metadata() {
 }
 
 up_to_3.2.0() {
-  # download 9.3.7 elastic agent packages
-  determine_elastic_agent_upgrade
-
   fix_logstash_0013_lumberjack_pipeline_name
 
   pin_elasticsearch_data_retention_method
@@ -972,7 +901,7 @@ post_to_3.2.0() {
 
   bootstrap_so_soc_database
 
-  # Generate 9.3.7 elastic agent installers
+  # Including agent regen script here since it was missed in post_to_3.1.0
   echo "Regenerating Elastic Agent Installers"
   /sbin/so-elastic-agent-gen-installers
 
@@ -980,7 +909,7 @@ post_to_3.2.0() {
 
   update_kafka_metadata "4.3"
 
-  set_postversion 3.2.0
+  POSTVERSION=3.2.0
 }
 
 ### 3.2.0 End ###
@@ -1132,20 +1061,8 @@ upgrade_check() {
   fi
   [[ -f /etc/sohotfix ]] && CURRENTHOTFIX=$(cat /etc/sohotfix)
   if [ "$INSTALLEDVERSION" == "$NEWVERSION" ]; then
-    # A leftover post-version marker means a previous upgrade to this version
-    # advanced /etc/soversion (the highstate stamps it from the pillar) but did not
-    # finish its post-upgrade steps. Resume the upgrade instead of reporting "latest".
-    if [ -f "$POSTVERSION_FILE" ] && [ "$(cat "$POSTVERSION_FILE")" != "$NEWVERSION" ]; then
-      echo "A previous upgrade to $NEWVERSION did not complete its post-upgrade steps; resuming."
-      is_hotfix=false
-      return 0
-    fi
     echo "Checking to see if there are hotfixes needed"
     if [ "$HOTFIXVERSION" == "$CURRENTHOTFIX" ]; then
-      # Reaching here means we are at the target version and NOT resuming (the resume
-      # check above returned otherwise). Clear any stale resume marker so a completed
-      # upgrade is never mistaken for a partial one and re-run on a later invocation.
-      rm -f "$POSTVERSION_FILE"
       echo "You are already running the latest version of Security Onion."
       exit 0
     else
@@ -1257,8 +1174,7 @@ verify_es_version_compatibility() {
         ["8.18.4"]="8.18.6 8.18.8 9.0.8"
 	    ["8.18.6"]="8.18.8 9.0.8"
 	    ["8.18.8"]="9.0.8"
-        ["9.0.8"]="9.3.3 9.3.7"
+        ["9.0.8"]="9.3.3"
-        ["9.3.3"]="9.3.7"
     )
 
     # Elasticsearch MUST upgrade through these versions
@@ -1841,15 +1757,6 @@ main() {
   set_minionid
   MINION_ROLE=$(lookup_role)
   echo "Found that Security Onion $INSTALLEDVERSION is currently installed."
-  # /etc/soversion is stamped to the target version before the upgrade fully
-  # completes, so a lingering resume marker means this grid is only partially
-  # upgraded even though the line above shows the target version. Make that explicit
-  # so it is not mistaken for a finished upgrade.
-  if [ -f "$POSTVERSION_FILE" ] && [ "$(cat "$POSTVERSION_FILE")" != "$INSTALLEDVERSION" ]; then
-    echo ""
-    echo "NOTE: A previous upgrade to $INSTALLEDVERSION did not finish. This grid is"
-    echo "      partially upgraded and this soup run will resume and complete it."
-  fi
   echo ""
   check_minimum_version
 
@@ -1878,12 +1785,6 @@ main() {
   echo "Verifying Elasticsearch version compatibility across the grid before upgrading."
   verify_es_version_compatibility
 
-  # Pre-flight health checks: confirm the grid is in a good state before we change
-  # anything. These run before any modifications, so a failure exits cleanly and the
-  # operator can fix the issue and re-run soup.
-  check_cluster_health
-  check_fleet_server
-
   echo "Checking for Salt Master and Minion updates."
   upgrade_check_salt
   set -e
@@ -1900,7 +1801,6 @@ main() {
   fi
 
   if [ "$is_hotfix" == "true" ]; then
-    SOUP_UPGRADE_STARTED=true
     echo "Applying $HOTFIXVERSION hotfix"
     # since we don't run the backup.config_backup state on import we wont snapshot previous version states and pillars
     if [[ ! "$MINION_ROLE" == "import" ]]; then
@@ -1911,16 +1811,10 @@ main() {
     create_local_directories "/opt/so/saltstack/default"
     apply_hotfix
     echo "Hotfix applied"
+    update_version
     enable_highstate
     highstate
-    # Record the hotfix only after the highstate succeeds. /etc/sohotfix is written
-    # solely by soup (no salt state manages it), so deferring the write means a failed
-    # hotfix highstate leaves the old hotfix value and re-running soup re-applies it,
-    # rather than reporting "already latest". The soversion/pillar writes in
-    # update_version are no-ops here since the version is unchanged for a hotfix.
-    update_version
   else
-    SOUP_UPGRADE_STARTED=true
     echo ""
     echo "Performing upgrade from Security Onion $INSTALLEDVERSION to Security Onion $NEWVERSION."
     echo ""
@@ -1976,10 +1870,6 @@ main() {
     copy_new_files
     echo ""
     create_local_directories "/opt/so/saltstack/default"
-    # Seed the resume marker before the highstate stamps /etc/soversion to the new
-    # version, so an interrupted upgrade is detectable as "not finished" on re-run.
-    # POSTVERSION still holds the pre-upgrade (or prior resume) version here.
-    [ -f "$POSTVERSION_FILE" ] || echo "$POSTVERSION" > "$POSTVERSION_FILE"
     update_version
 
     echo ""

salt/nginx/enabled.sls

@@ -8,6 +8,7 @@
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {%   from 'nginx/map.jinja' import NGINXMERGED %}
+{%   from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
 
 include:
   - nginx.ssl
@@ -31,6 +32,8 @@ make-rule-dir-nginx:
 {%     set container_config = 'so-nginx-fleet-node' %}
 {%   endif %}
 
+{{ clear_stale_endpoint('so-nginx', 'sobridge', DOCKERMERGED.containers[container_config].ip) }}
+
 so-nginx:
   docker_container.running:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-nginx:{{ GLOBALS.so_version }}

salt/nginx/etc/nginx.conf

@@ -399,7 +399,7 @@ http {
 		error_page 429 = @error429;
 
 		location @error401 {
-			if ($request_uri ~* (^.*/api/.*|^/connect/.*|^/oauth2/.*|^/.*\.map$)) {
+			if ($request_uri ~* (^/api/.*|^/connect/.*|^/oauth2/.*|^/.*\.map$)) {
 				return    401;
 			}
 

salt/postgres/enabled.sls

@@ -8,6 +8,9 @@
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {%   set SO_POSTGRES_USER = salt['pillar.get']('postgres:auth:users:so_postgres_user:user', 'so_postgres') %}
+{%   from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
+
+{{ clear_stale_endpoint('so-postgres', 'sobridge', DOCKERMERGED.containers['so-postgres'].ip) }}
 
 include:
   - postgres.auth

salt/redis/enabled.sls

@@ -7,6 +7,9 @@
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
+
+{{ clear_stale_endpoint('so-redis', 'sobridge', DOCKERMERGED.containers['so-redis'].ip) }}
 
 include:
   - ca

salt/registry/enabled.sls

@@ -7,6 +7,9 @@
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
+{%   from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
+
+{{ clear_stale_endpoint('so-dockerregistry', 'sobridge', DOCKERMERGED.containers['so-dockerregistry'].ip) }}
 
 include:
   - registry.ssl

salt/soc/enabled.sls

@@ -10,6 +10,9 @@
 {%   from 'soc/merged.map.jinja' import DOCKER_EXTRA_HOSTS %}
 {%   from 'soc/merged.map.jinja' import SOCMERGED %}
 
+{% from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
+{{ clear_stale_endpoint('so-soc', 'sobridge', DOCKERMERGED.containers['so-soc'].ip) }}
+
 include:
   - ca
   - soc.config

salt/strelka/backend/enabled.sls

@@ -7,6 +7,9 @@
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
+
+{{ clear_stale_endpoint('so-strelka-backend', 'sobridge', DOCKERMERGED.containers['so-strelka-backend'].ip, state_id='strelka_backend') }}
 
 include:
   - strelka.backend.config

salt/strelka/coordinator/enabled.sls

@@ -7,6 +7,9 @@
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
+
+{{ clear_stale_endpoint('so-strelka-coordinator', 'sobridge', DOCKERMERGED.containers['so-strelka-coordinator'].ip, state_id='strelka_coordinator') }}
 
 include:
   - strelka.coordinator.config

salt/strelka/filestream/enabled.sls

@@ -7,6 +7,9 @@
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
+
+{{ clear_stale_endpoint('so-strelka-filestream', 'sobridge', DOCKERMERGED.containers['so-strelka-filestream'].ip, state_id='strelka_filestream') }}
 
 include:
   - strelka.filestream.config

salt/strelka/frontend/enabled.sls

@@ -7,6 +7,9 @@
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
+
+{{ clear_stale_endpoint('so-strelka-frontend', 'sobridge', DOCKERMERGED.containers['so-strelka-frontend'].ip, state_id='strelka_frontend') }}
 
 include:
   - strelka.frontend.config

salt/strelka/gatekeeper/enabled.sls

@@ -7,6 +7,9 @@
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
+
+{{ clear_stale_endpoint('so-strelka-gatekeeper', 'sobridge', DOCKERMERGED.containers['so-strelka-gatekeeper'].ip, state_id='strelka_gatekeeper') }}
 
 include:
   - strelka.gatekeeper.config

salt/strelka/manager/enabled.sls

@@ -7,6 +7,9 @@
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
+
+{{ clear_stale_endpoint('so-strelka-manager', 'sobridge', DOCKERMERGED.containers['so-strelka-manager'].ip, state_id='strelka_manager') }}
 
 include:
   - strelka.manager.config