Compare commits

..
Author SHA1 Message Date
Josh Patterson cd8f7f99e3 clear stale docker network endpoints before starting containers
docker_container.running fails with an address-already-in-use error when a
container holds a stale sobridge endpoint - either after its IP changes in
pillar or after an unclean shutdown leaves the endpoint behind.

Add the clear_stale_endpoint macro and call it from every enabled.sls whose
container attaches to sobridge with a static IP. The macro force-disconnects
the container from the network only when its current endpoint IP does not
match the one salt is about to assign, so it is a no-op on a normal highstate.

The strelka states use state IDs that differ from their container names, so
those calls pass state_id explicitly. nginx resolves its IP through
container_config, which varies by role.
2026-07-13 15:11:42 -04:00
32 changed files with 97 additions and 21 deletions
+12
View File
@@ -0,0 +1,12 @@
{% macro clear_stale_endpoint(container, network, ipv4, state_id=None) %}
{{ container }}_{{ network }}_stale_endpoint:
cmd.run:
- name: docker network disconnect -f {{ network }} {{ container }} || true
- onlyif: docker inspect {{ container }}
- unless: >-
docker inspect -f
'{{ '{{' }} with index .NetworkSettings.Networks "{{ network }}" {{ '}}' }}{{ '{{' }} .IPAMConfig.IPv4Address {{ '}}' }}{{ '{{' }} end {{ '}}' }}'
{{ container }} 2>/dev/null | grep -qx '{{ ipv4 }}'
- require_in:
- docker_container: {{ state_id or container }}
{% endmacro %}
+3
View File
@@ -7,6 +7,9 @@
{% if sls.split('.')[0] in allowed_states %}
{% from 'vars/globals.map.jinja' import GLOBALS %}
{% from 'docker/docker.map.jinja' import DOCKERMERGED %}
{% from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
{{ clear_stale_endpoint('so-elastalert', 'sobridge', DOCKERMERGED.containers['so-elastalert'].ip) }}
include:
- elastalert.config
@@ -7,6 +7,9 @@
{% if sls.split('.')[0] in allowed_states %}
{% from 'vars/globals.map.jinja' import GLOBALS %}
{% from 'docker/docker.map.jinja' import DOCKERMERGED %}
{% from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
{{ clear_stale_endpoint('so-elastic-fleet-package-registry', 'sobridge', DOCKERMERGED.containers['so-elastic-fleet-package-registry'].ip) }}
include:
- elastic-fleet-package-registry.config
+3
View File
@@ -7,6 +7,9 @@
{% if sls.split('.')[0] in allowed_states %}
{% from 'vars/globals.map.jinja' import GLOBALS %}
{% from 'docker/docker.map.jinja' import DOCKERMERGED %}
{% from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
{{ clear_stale_endpoint('so-elastic-agent', 'sobridge', DOCKERMERGED.containers['so-elastic-agent'].ip) }}
include:
- ca
+1
View File
@@ -1,5 +1,6 @@
elasticfleet:
enabled: False
patch_version: 9.3.3+build202604082258 # Elastic Agent specific patch release.
enable_manager_output: True
config:
server:
+3
View File
@@ -8,6 +8,7 @@
{% from 'vars/globals.map.jinja' import GLOBALS %}
{% from 'docker/docker.map.jinja' import DOCKERMERGED %}
{% from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %}
{% from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
{# This value is generated during node install and stored in minion pillar #}
{% set SERVICETOKEN = salt['pillar.get']('elasticfleet:config:server:es_token','') %}
@@ -43,6 +44,8 @@ elasticagent_syncartifacts:
{% endif %}
{% if SERVICETOKEN != '' %}
{{ clear_stale_endpoint('so-elastic-fleet', 'sobridge', DOCKERMERGED.containers['so-elastic-fleet'].ip) }}
so-elastic-fleet:
docker_container.running:
- image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent:{{ GLOBALS.so_version }}
@@ -5,7 +5,7 @@
"package": {
"name": "endpoint",
"title": "Elastic Defend",
"version": "9.3.1",
"version": "9.3.0",
"requires_root": true
},
"enabled": true,
@@ -29,7 +29,7 @@
"\\.gz$"
],
"include_files": [],
"processors": "- dissect:\n tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"\n- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n- drop_fields:\n fields: [\"host\"]\n ignore_missing: true\n- add_fields:\n target: data_stream\n fields:\n type: logs\n dataset: system.security\n- add_fields:\n target: event\n fields:\n dataset: system.security\n module: system\n imported: true\n- add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.security-2.20.0\n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.sysmon_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.sysmon_operational\n module: windows\n imported: true\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.sysmon_operational-3.8.3\n- if:\n equals:\n winlog.channel: 'Application'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.application\n - add_fields:\n target: event\n fields:\n dataset: system.application\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.application-2.20.0\n- if:\n equals:\n winlog.channel: 'System'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.system\n - add_fields:\n target: event\n fields:\n dataset: system.system\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.system-2.20.0\n \n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.powershell_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.powershell_operational\n module: windows\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.powershell_operational-3.8.3\n- add_fields:\n target: data_stream\n fields:\n dataset: import",
"processors": "- dissect:\n tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"\n- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n- drop_fields:\n fields: [\"host\"]\n ignore_missing: true\n- add_fields:\n target: data_stream\n fields:\n type: logs\n dataset: system.security\n- add_fields:\n target: event\n fields:\n dataset: system.security\n module: system\n imported: true\n- add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.security-2.15.0\n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.sysmon_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.sysmon_operational\n module: windows\n imported: true\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.sysmon_operational-3.8.0\n- if:\n equals:\n winlog.channel: 'Application'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.application\n - add_fields:\n target: event\n fields:\n dataset: system.application\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.application-2.15.0\n- if:\n equals:\n winlog.channel: 'System'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.system\n - add_fields:\n target: event\n fields:\n dataset: system.system\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.system-2.15.0\n \n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.powershell_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.powershell_operational\n module: windows\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.powershell_operational-3.8.0\n- add_fields:\n target: data_stream\n fields:\n dataset: import",
"tags": [
"import"
],
+1 -1
View File
@@ -1,6 +1,6 @@
elasticsearch:
enabled: false
version: 9.3.7
version: 9.3.3
index_clean: true
data_retention_method: DLM
vm:
+3
View File
@@ -10,6 +10,9 @@
{% from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_NODES %}
{% from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_SEED_HOSTS %}
{% from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
{% from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
{{ clear_stale_endpoint('so-elasticsearch', 'sobridge', DOCKERMERGED.containers['so-elasticsearch'].ip) }}
include:
- ca
@@ -118,70 +118,70 @@
{
"pipeline": {
"tag": "pipeline_e16851a7",
"name": "logs-pfsense.log-1.25.4-firewall",
"name": "logs-pfsense.log-1.25.2-firewall",
"if": "ctx.event.provider == 'filterlog'"
}
},
{
"pipeline": {
"tag": "pipeline_828590b5",
"name": "logs-pfsense.log-1.25.4-openvpn",
"name": "logs-pfsense.log-1.25.2-openvpn",
"if": "ctx.event.provider == 'openvpn'"
}
},
{
"pipeline": {
"tag": "pipeline_9d37039c",
"name": "logs-pfsense.log-1.25.4-ipsec",
"name": "logs-pfsense.log-1.25.2-ipsec",
"if": "ctx.event.provider == 'charon'"
}
},
{
"pipeline": {
"tag": "pipeline_ad56bbca",
"name": "logs-pfsense.log-1.25.4-dhcp",
"name": "logs-pfsense.log-1.25.2-dhcp",
"if": "[\"dhcpd\", \"dhclient\", \"dhcp6c\", \"dnsmasq-dhcp\"].contains(ctx.event.provider)"
}
},
{
"pipeline": {
"tag": "pipeline_dd85553d",
"name": "logs-pfsense.log-1.25.4-unbound",
"name": "logs-pfsense.log-1.25.2-unbound",
"if": "ctx.event.provider == 'unbound'"
}
},
{
"pipeline": {
"tag": "pipeline_720ed255",
"name": "logs-pfsense.log-1.25.4-haproxy",
"name": "logs-pfsense.log-1.25.2-haproxy",
"if": "ctx.event.provider == 'haproxy'"
}
},
{
"pipeline": {
"tag": "pipeline_456beba5",
"name": "logs-pfsense.log-1.25.4-php-fpm",
"name": "logs-pfsense.log-1.25.2-php-fpm",
"if": "ctx.event.provider == 'php-fpm'"
}
},
{
"pipeline": {
"tag": "pipeline_a0d89375",
"name": "logs-pfsense.log-1.25.4-squid",
"name": "logs-pfsense.log-1.25.2-squid",
"if": "ctx.event.provider == 'squid'"
}
},
{
"pipeline": {
"tag": "pipeline_c2f1ed55",
"name": "logs-pfsense.log-1.25.4-snort",
"name": "logs-pfsense.log-1.25.2-snort",
"if": "ctx.event.provider == 'snort'"
}
},
{
"pipeline": {
"tag":"pipeline_33db1c9e",
"name": "logs-pfsense.log-1.25.4-suricata",
"name": "logs-pfsense.log-1.25.2-suricata",
"if": "ctx.event.provider == 'suricata'"
}
},
+3
View File
@@ -14,6 +14,9 @@
{% from 'docker/docker.map.jinja' import DOCKERMERGED %}
{% from 'vars/globals.map.jinja' import GLOBALS %}
{% if 'api' in salt['pillar.get']('features', []) %}
{% from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
{{ clear_stale_endpoint('so-hydra', 'sobridge', DOCKERMERGED.containers['so-hydra'].ip) }}
include:
- hydra.config
+3
View File
@@ -9,6 +9,9 @@
{% from 'docker/docker.map.jinja' import DOCKERMERGED %}
{% set PASSWORD = salt['pillar.get']('secrets:influx_pass') %}
{% set TOKEN = salt['pillar.get']('influxdb:token') %}
{% from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
{{ clear_stale_endpoint('so-influxdb', 'sobridge', DOCKERMERGED.containers['so-influxdb'].ip) }}
include:
- influxdb.ssl
+3
View File
@@ -16,6 +16,9 @@
{% set KAFKANODES = salt['pillar.get']('kafka:nodes') %}
{% set KAFKA_EXTERNAL_ACCESS = salt['pillar.get']('kafka:config:external_access:enabled', default=False) %}
{% if 'gmd' in salt['pillar.get']('features', []) %}
{% from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
{{ clear_stale_endpoint('so-kafka', 'sobridge', DOCKERMERGED.containers['so-kafka'].ip) }}
include:
- kafka.ca
+1 -1
View File
@@ -22,7 +22,7 @@ kibana:
- default
- file
migrations:
discardCorruptObjects: "9.3.7"
discardCorruptObjects: "9.3.3"
telemetry:
enabled: False
xpack:
+3
View File
@@ -8,6 +8,9 @@
{% from 'docker/docker.map.jinja' import DOCKERMERGED %}
{% from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
{% from 'vars/globals.map.jinja' import GLOBALS %}
{% from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
{{ clear_stale_endpoint('so-kibana', 'sobridge', DOCKERMERGED.containers['so-kibana'].ip) }}
include:
- kibana.config
+3
View File
@@ -7,6 +7,9 @@
{% if sls.split('.')[0] in allowed_states %}
{% from 'docker/docker.map.jinja' import DOCKERMERGED %}
{% from 'vars/globals.map.jinja' import GLOBALS %}
{% from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
{{ clear_stale_endpoint('so-kratos', 'sobridge', DOCKERMERGED.containers['so-kratos'].ip) }}
include:
- kratos.config
+3
View File
@@ -10,6 +10,9 @@
{% from 'logstash/map.jinja' import LOGSTASH_MERGED %}
{% from 'logstash/map.jinja' import LOGSTASH_NODES %}
{% set lsheap = LOGSTASH_MERGED.settings.lsheap %}
{% from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
{{ clear_stale_endpoint('so-logstash', 'sobridge', DOCKERMERGED.containers['so-logstash'].ip) }}
include:
- ca
+3 -6
View File
@@ -740,6 +740,7 @@ fix_logstash_0013_lumberjack_pipeline_name() {
up_to_3.1.0() {
ensure_postgres_local_pillar
ensure_postgres_secret
determine_elastic_agent_upgrade
elasticsearch_backup_index_templates
# Clear existing component template state file.
rm -f /opt/so/state/esfleet_component_templates.json
@@ -887,9 +888,6 @@ update_kafka_metadata() {
}
up_to_3.2.0() {
# download 9.3.7 elastic agent packages
determine_elastic_agent_upgrade
fix_logstash_0013_lumberjack_pipeline_name
pin_elasticsearch_data_retention_method
@@ -903,7 +901,7 @@ post_to_3.2.0() {
bootstrap_so_soc_database
# Generate 9.3.7 elastic agent installers
# Including agent regen script here since it was missed in post_to_3.1.0
echo "Regenerating Elastic Agent Installers"
/sbin/so-elastic-agent-gen-installers
@@ -1176,8 +1174,7 @@ verify_es_version_compatibility() {
["8.18.4"]="8.18.6 8.18.8 9.0.8"
["8.18.6"]="8.18.8 9.0.8"
["8.18.8"]="9.0.8"
["9.0.8"]="9.3.3 9.3.7"
["9.3.3"]="9.3.7"
["9.0.8"]="9.3.3"
)
# Elasticsearch MUST upgrade through these versions
+3
View File
@@ -8,6 +8,7 @@
{% from 'vars/globals.map.jinja' import GLOBALS %}
{% from 'docker/docker.map.jinja' import DOCKERMERGED %}
{% from 'nginx/map.jinja' import NGINXMERGED %}
{% from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
include:
- nginx.ssl
@@ -31,6 +32,8 @@ make-rule-dir-nginx:
{% set container_config = 'so-nginx-fleet-node' %}
{% endif %}
{{ clear_stale_endpoint('so-nginx', 'sobridge', DOCKERMERGED.containers[container_config].ip) }}
so-nginx:
docker_container.running:
- image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-nginx:{{ GLOBALS.so_version }}
+1 -1
View File
@@ -399,7 +399,7 @@ http {
error_page 429 = @error429;
location @error401 {
if ($request_uri ~* (^.*/api/.*|^/connect/.*|^/oauth2/.*|^/.*\.map$)) {
if ($request_uri ~* (^/api/.*|^/connect/.*|^/oauth2/.*|^/.*\.map$)) {
return 401;
}
+3
View File
@@ -8,6 +8,9 @@
{% from 'vars/globals.map.jinja' import GLOBALS %}
{% from 'docker/docker.map.jinja' import DOCKERMERGED %}
{% set SO_POSTGRES_USER = salt['pillar.get']('postgres:auth:users:so_postgres_user:user', 'so_postgres') %}
{% from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
{{ clear_stale_endpoint('so-postgres', 'sobridge', DOCKERMERGED.containers['so-postgres'].ip) }}
include:
- postgres.auth
+3
View File
@@ -7,6 +7,9 @@
{% if sls.split('.')[0] in allowed_states %}
{% from 'docker/docker.map.jinja' import DOCKERMERGED %}
{% from 'vars/globals.map.jinja' import GLOBALS %}
{% from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
{{ clear_stale_endpoint('so-redis', 'sobridge', DOCKERMERGED.containers['so-redis'].ip) }}
include:
- ca
+3
View File
@@ -7,6 +7,9 @@
{% if sls.split('.')[0] in allowed_states %}
{% from 'vars/globals.map.jinja' import GLOBALS %}
{% from 'docker/docker.map.jinja' import DOCKERMERGED %}
{% from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
{{ clear_stale_endpoint('so-dockerregistry', 'sobridge', DOCKERMERGED.containers['so-dockerregistry'].ip) }}
include:
- registry.ssl
+3
View File
@@ -10,6 +10,9 @@
{% from 'soc/merged.map.jinja' import DOCKER_EXTRA_HOSTS %}
{% from 'soc/merged.map.jinja' import SOCMERGED %}
{% from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
{{ clear_stale_endpoint('so-soc', 'sobridge', DOCKERMERGED.containers['so-soc'].ip) }}
include:
- ca
- soc.config
+3
View File
@@ -7,6 +7,9 @@
{% if sls.split('.')[0] in allowed_states %}
{% from 'docker/docker.map.jinja' import DOCKERMERGED %}
{% from 'vars/globals.map.jinja' import GLOBALS %}
{% from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
{{ clear_stale_endpoint('so-strelka-backend', 'sobridge', DOCKERMERGED.containers['so-strelka-backend'].ip, state_id='strelka_backend') }}
include:
- strelka.backend.config
+3
View File
@@ -7,6 +7,9 @@
{% if sls.split('.')[0] in allowed_states %}
{% from 'docker/docker.map.jinja' import DOCKERMERGED %}
{% from 'vars/globals.map.jinja' import GLOBALS %}
{% from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
{{ clear_stale_endpoint('so-strelka-coordinator', 'sobridge', DOCKERMERGED.containers['so-strelka-coordinator'].ip, state_id='strelka_coordinator') }}
include:
- strelka.coordinator.config
+3
View File
@@ -7,6 +7,9 @@
{% if sls.split('.')[0] in allowed_states %}
{% from 'docker/docker.map.jinja' import DOCKERMERGED %}
{% from 'vars/globals.map.jinja' import GLOBALS %}
{% from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
{{ clear_stale_endpoint('so-strelka-filestream', 'sobridge', DOCKERMERGED.containers['so-strelka-filestream'].ip, state_id='strelka_filestream') }}
include:
- strelka.filestream.config
+3
View File
@@ -7,6 +7,9 @@
{% if sls.split('.')[0] in allowed_states %}
{% from 'docker/docker.map.jinja' import DOCKERMERGED %}
{% from 'vars/globals.map.jinja' import GLOBALS %}
{% from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
{{ clear_stale_endpoint('so-strelka-frontend', 'sobridge', DOCKERMERGED.containers['so-strelka-frontend'].ip, state_id='strelka_frontend') }}
include:
- strelka.frontend.config
+3
View File
@@ -7,6 +7,9 @@
{% if sls.split('.')[0] in allowed_states %}
{% from 'docker/docker.map.jinja' import DOCKERMERGED %}
{% from 'vars/globals.map.jinja' import GLOBALS %}
{% from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
{{ clear_stale_endpoint('so-strelka-gatekeeper', 'sobridge', DOCKERMERGED.containers['so-strelka-gatekeeper'].ip, state_id='strelka_gatekeeper') }}
include:
- strelka.gatekeeper.config
+3
View File
@@ -7,6 +7,9 @@
{% if sls.split('.')[0] in allowed_states %}
{% from 'docker/docker.map.jinja' import DOCKERMERGED %}
{% from 'vars/globals.map.jinja' import GLOBALS %}
{% from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
{{ clear_stale_endpoint('so-strelka-manager', 'sobridge', DOCKERMERGED.containers['so-strelka-manager'].ip, state_id='strelka_manager') }}
include:
- strelka.manager.config