clear stale docker network endpoints before starting containers

docker_container.running fails with an address-already-in-use error when a
container holds a stale sobridge endpoint - either after its IP changes in
pillar or after an unclean shutdown leaves the endpoint behind.

Add the clear_stale_endpoint macro and call it from every enabled.sls whose
container attaches to sobridge with a static IP. The macro force-disconnects
the container from the network only when its current endpoint IP does not
match the one salt is about to assign, so it is a no-op on a normal highstate.

The strelka states use state IDs that differ from their container names, so
those calls pass state_id explicitly. nginx resolves its IP through
container_config, which varies by role.

salt/docker/macros/docker_endpoint.jinja

@@ -0,0 +1,12 @@
+{% macro clear_stale_endpoint(container, network, ipv4, state_id=None) %}
+{{ container }}_{{ network }}_stale_endpoint:
+  cmd.run:
+    - name: docker network disconnect -f {{ network }} {{ container }} || true
+    - onlyif: docker inspect {{ container }}
+    - unless: >-
+        docker inspect -f
+        '{{ '{{' }} with index .NetworkSettings.Networks "{{ network }}" {{ '}}' }}{{ '{{' }} .IPAMConfig.IPv4Address {{ '}}' }}{{ '{{' }} end {{ '}}' }}'
+        {{ container }} 2>/dev/null | grep -qx '{{ ipv4 }}'
+    - require_in:
+      - docker_container: {{ state_id or container }}
+{% endmacro %}

salt/elastalert/enabled.sls

@@ -7,6 +7,9 @@
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
+{%   from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
+
+{{ clear_stale_endpoint('so-elastalert', 'sobridge', DOCKERMERGED.containers['so-elastalert'].ip) }}
 
 include:
   - elastalert.config

salt/elastic-fleet-package-registry/enabled.sls

@@ -7,6 +7,9 @@
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
+{%   from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
+
+{{ clear_stale_endpoint('so-elastic-fleet-package-registry', 'sobridge', DOCKERMERGED.containers['so-elastic-fleet-package-registry'].ip) }}
 
 include:
   - elastic-fleet-package-registry.config

salt/elasticagent/enabled.sls

@@ -7,6 +7,9 @@
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
+{%   from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
+
+{{ clear_stale_endpoint('so-elastic-agent', 'sobridge', DOCKERMERGED.containers['so-elastic-agent'].ip) }}
 
 include:
   - ca

salt/elasticfleet/enabled.sls

@@ -8,6 +8,7 @@
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {%   from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %}
+{%   from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
 
 {#   This value is generated during node install and stored in minion pillar #}
 {%   set SERVICETOKEN = salt['pillar.get']('elasticfleet:config:server:es_token','') %}
@@ -43,6 +44,8 @@ elasticagent_syncartifacts:
 {% endif %}
 
 {%   if SERVICETOKEN != '' %}
+{{ clear_stale_endpoint('so-elastic-fleet', 'sobridge', DOCKERMERGED.containers['so-elastic-fleet'].ip) }}
+
 so-elastic-fleet:
   docker_container.running:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent:{{ GLOBALS.so_version }}

salt/elasticsearch/enabled.sls

@@ -10,6 +10,9 @@
 {%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_NODES %}
 {%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_SEED_HOSTS %}
 {%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
+{%   from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
+
+{{ clear_stale_endpoint('so-elasticsearch', 'sobridge', DOCKERMERGED.containers['so-elasticsearch'].ip) }}
 
 include:
   - ca

salt/hydra/enabled.sls

@@ -14,6 +14,9 @@
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 {%   if 'api' in salt['pillar.get']('features', []) %}
+{%   from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
+
+{{ clear_stale_endpoint('so-hydra', 'sobridge', DOCKERMERGED.containers['so-hydra'].ip) }}
 
 include:
   - hydra.config

salt/influxdb/enabled.sls

@@ -9,6 +9,9 @@
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {%   set PASSWORD = salt['pillar.get']('secrets:influx_pass') %}
 {%   set TOKEN = salt['pillar.get']('influxdb:token') %}
+{%   from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
+
+{{ clear_stale_endpoint('so-influxdb', 'sobridge', DOCKERMERGED.containers['so-influxdb'].ip) }}
 
 include:
   - influxdb.ssl

salt/kafka/enabled.sls

@@ -16,6 +16,9 @@
 {%   set KAFKANODES = salt['pillar.get']('kafka:nodes') %}
 {%   set KAFKA_EXTERNAL_ACCESS = salt['pillar.get']('kafka:config:external_access:enabled', default=False) %}
 {%   if 'gmd' in salt['pillar.get']('features', []) %}
+{%   from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
+
+{{ clear_stale_endpoint('so-kafka', 'sobridge', DOCKERMERGED.containers['so-kafka'].ip) }}
 
 include:
   - kafka.ca

salt/kibana/enabled.sls

@@ -8,6 +8,9 @@
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
+
+{{ clear_stale_endpoint('so-kibana', 'sobridge', DOCKERMERGED.containers['so-kibana'].ip) }}
 
 include:
   - kibana.config

salt/kratos/enabled.sls

@@ -7,6 +7,9 @@
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
+
+{{ clear_stale_endpoint('so-kratos', 'sobridge', DOCKERMERGED.containers['so-kratos'].ip) }}
 
 include:
   - kratos.config

salt/logstash/enabled.sls

@@ -10,6 +10,9 @@
 {%   from 'logstash/map.jinja' import LOGSTASH_MERGED %}
 {%   from 'logstash/map.jinja' import LOGSTASH_NODES %}
 {%   set lsheap = LOGSTASH_MERGED.settings.lsheap %}
+{%   from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
+
+{{ clear_stale_endpoint('so-logstash', 'sobridge', DOCKERMERGED.containers['so-logstash'].ip) }}
 
 include:
   - ca

salt/nginx/enabled.sls

@@ -8,6 +8,7 @@
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {%   from 'nginx/map.jinja' import NGINXMERGED %}
+{%   from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
 
 include:
   - nginx.ssl
@@ -31,6 +32,8 @@ make-rule-dir-nginx:
 {%     set container_config = 'so-nginx-fleet-node' %}
 {%   endif %}
 
+{{ clear_stale_endpoint('so-nginx', 'sobridge', DOCKERMERGED.containers[container_config].ip) }}
+
 so-nginx:
   docker_container.running:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-nginx:{{ GLOBALS.so_version }}

salt/postgres/enabled.sls

@@ -8,6 +8,9 @@
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {%   set SO_POSTGRES_USER = salt['pillar.get']('postgres:auth:users:so_postgres_user:user', 'so_postgres') %}
+{%   from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
+
+{{ clear_stale_endpoint('so-postgres', 'sobridge', DOCKERMERGED.containers['so-postgres'].ip) }}
 
 include:
   - postgres.auth

salt/redis/enabled.sls

@@ -7,6 +7,9 @@
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
+
+{{ clear_stale_endpoint('so-redis', 'sobridge', DOCKERMERGED.containers['so-redis'].ip) }}
 
 include:
   - ca

salt/registry/enabled.sls

@@ -7,6 +7,9 @@
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
+{%   from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
+
+{{ clear_stale_endpoint('so-dockerregistry', 'sobridge', DOCKERMERGED.containers['so-dockerregistry'].ip) }}
 
 include:
   - registry.ssl

salt/soc/enabled.sls

@@ -10,6 +10,9 @@
 {%   from 'soc/merged.map.jinja' import DOCKER_EXTRA_HOSTS %}
 {%   from 'soc/merged.map.jinja' import SOCMERGED %}
 
+{% from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
+{{ clear_stale_endpoint('so-soc', 'sobridge', DOCKERMERGED.containers['so-soc'].ip) }}
+
 include:
   - ca
   - soc.config

salt/strelka/backend/enabled.sls

@@ -7,6 +7,9 @@
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
+
+{{ clear_stale_endpoint('so-strelka-backend', 'sobridge', DOCKERMERGED.containers['so-strelka-backend'].ip, state_id='strelka_backend') }}
 
 include:
   - strelka.backend.config

salt/strelka/coordinator/enabled.sls

@@ -7,6 +7,9 @@
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
+
+{{ clear_stale_endpoint('so-strelka-coordinator', 'sobridge', DOCKERMERGED.containers['so-strelka-coordinator'].ip, state_id='strelka_coordinator') }}
 
 include:
   - strelka.coordinator.config

salt/strelka/filestream/enabled.sls

@@ -7,6 +7,9 @@
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
+
+{{ clear_stale_endpoint('so-strelka-filestream', 'sobridge', DOCKERMERGED.containers['so-strelka-filestream'].ip, state_id='strelka_filestream') }}
 
 include:
   - strelka.filestream.config

salt/strelka/frontend/enabled.sls

@@ -7,6 +7,9 @@
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
+
+{{ clear_stale_endpoint('so-strelka-frontend', 'sobridge', DOCKERMERGED.containers['so-strelka-frontend'].ip, state_id='strelka_frontend') }}
 
 include:
   - strelka.frontend.config

salt/strelka/gatekeeper/enabled.sls

@@ -7,6 +7,9 @@
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
+
+{{ clear_stale_endpoint('so-strelka-gatekeeper', 'sobridge', DOCKERMERGED.containers['so-strelka-gatekeeper'].ip, state_id='strelka_gatekeeper') }}
 
 include:
   - strelka.gatekeeper.config

salt/strelka/manager/enabled.sls

@@ -7,6 +7,9 @@
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
+
+{{ clear_stale_endpoint('so-strelka-manager', 'sobridge', DOCKERMERGED.containers['so-strelka-manager'].ip, state_id='strelka_manager') }}
 
 include:
   - strelka.manager.config