From cf3a4ebc27da63a3ee239d3df19369d39ebb302d Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 9 Sep 2026 13:08:41 -0400 Subject: [PATCH 1/3] Isolate the Kratos admin API on a dedicated soauth docker network The Kratos admin API is unauthenticated by design and relies on network isolation, but so-kratos published 0.0.0.0:4434:4434 and daemon.json leaves userland-proxy at its default of true. Docker therefore ran a proxy listener on the host, and any container reached the admin API through the manager IP or the bridge gateway regardless of its docker network. so-kratos now sits alone on a new soauth network and no longer publishes 4434. 4433 is still published, so nginx is unchanged. so-soc is dual homed and reaches the admin API over soauth, with publicHostUrl set explicitly. so-user reaches it through docker exec, and wait_for_kratos polls the container address instead of the host port. firewall/iptables.jinja hardcoded sobridge in every generated DNAT, ACCEPT, masquerade and isolation rule, so it is now driven by a docker:networks map and a per-container networks list. Containers that do not declare one default to sobridge, and .ip still resolves to the primary network's address. soauth is only created on the roles that run so-kratos. Setup already allows a custom docker range, so it now also prompts for the auth network range and writes it to the docker pillar. Grids on the default range need no decision during soup, since they get the same 172.17.2.0/24 a fresh install would. Grids that set a custom range are prompted, defaulting to the adjacent /24, and unattended upgrades take that default with a notice rather than blocking. so-kratos and so-soc are removed in up_to_3.4.0 so the highstate recreates them with the correct network membership. --- salt/docker/defaults.yaml | 9 +++- salt/docker/docker.map.jinja | 24 ++++++++-- salt/docker/init.sls | 16 ++++--- salt/docker/soc_docker.yaml | 20 +++++++++ salt/firewall/iptables.jinja | 47 ++++++++++++++------ salt/firewall/map.jinja | 8 +++- salt/kratos/enabled.sls | 6 +-- salt/manager/tools/sbin/so-user | 27 ++++++----- salt/manager/tools/sbin/soup | 79 +++++++++++++++++++++++++++++++++ salt/soc/defaults.map.jinja | 2 + salt/soc/defaults.yaml | 1 + salt/soc/enabled.sls | 4 +- salt/vars/globals.map.jinja | 2 +- setup/so-functions | 20 +++++++++ setup/so-whiptail | 13 ++++++ 15 files changed, 236 insertions(+), 42 deletions(-) diff --git a/salt/docker/defaults.yaml b/salt/docker/defaults.yaml index 81ff07190..ea656f3dd 100644 --- a/salt/docker/defaults.yaml +++ b/salt/docker/defaults.yaml @@ -1,6 +1,12 @@ docker: range: '172.17.1.0/24' gateway: '172.17.1.1' + networks: + sobridge: {} + soauth: + range: '172.17.2.0/24' + gateway: '172.17.2.1' + manager_only: True ulimits: - name: nofile soft: 1048576 @@ -58,9 +64,9 @@ docker: ulimits: [] 'so-kratos': final_octet: 28 + networks: ['soauth'] port_bindings: - 0.0.0.0:4433:4433 - - 0.0.0.0:4434:4434 custom_bind_mounts: [] extra_hosts: [] extra_env: [] @@ -128,6 +134,7 @@ docker: ulimits: [] 'so-soc': final_octet: 34 + networks: ['sobridge', 'soauth'] port_bindings: - 0.0.0.0:9822:9822 custom_bind_mounts: [] diff --git a/salt/docker/docker.map.jinja b/salt/docker/docker.map.jinja index 595d5d272..01051cb6d 100644 --- a/salt/docker/docker.map.jinja +++ b/salt/docker/docker.map.jinja @@ -1,8 +1,26 @@ {% import_yaml 'docker/defaults.yaml' as DOCKERDEFAULTS %} {% set DOCKERMERGED = salt['pillar.get']('docker', DOCKERDEFAULTS.docker, merge=True) %} -{% set RANGESPLIT = DOCKERMERGED.range.split('.') %} -{% set FIRSTTHREE = RANGESPLIT[0] ~ '.' ~ RANGESPLIT[1] ~ '.' ~ RANGESPLIT[2] ~ '.' %} + +{% if DOCKERMERGED.networks.sobridge is not mapping %} +{% do DOCKERMERGED.networks.update({'sobridge': {}}) %} +{% endif %} +{% do DOCKERMERGED.networks['sobridge'].update({'range': DOCKERMERGED.range, 'gateway': DOCKERMERGED.gateway}) %} + +{% for netname, net in DOCKERMERGED.networks.items() %} +{% set RANGESPLIT = net.range.split('.') %} +{% do net.update({'prefix': RANGESPLIT[0] ~ '.' ~ RANGESPLIT[1] ~ '.' ~ RANGESPLIT[2] ~ '.'}) %} +{% endfor %} {% for container, vals in DOCKERMERGED.containers.items() %} -{% do DOCKERMERGED.containers[container].update({'ip': FIRSTTHREE ~ DOCKERMERGED.containers[container].final_octet}) %} +{% set CONTAINER_NETS = vals.get('networks', ['sobridge']) %} +{% set IPS = {} %} +{% for netname in CONTAINER_NETS %} +{% do IPS.update({netname: DOCKERMERGED.networks[netname].prefix ~ vals.final_octet}) %} +{% endfor %} +{% do DOCKERMERGED.containers[container].update({ + 'networks': CONTAINER_NETS, + 'ips': IPS, + 'network': CONTAINER_NETS[0], + 'ip': IPS[CONTAINER_NETS[0]] + }) %} {% endfor %} diff --git a/salt/docker/init.sls b/salt/docker/init.sls index 52091ed95..9945a8096 100644 --- a/salt/docker/init.sls +++ b/salt/docker/init.sls @@ -71,15 +71,19 @@ dockerreserveports: - source: salt://common/files/99-reserved-ports.conf - name: /etc/sysctl.d/99-reserved-ports.conf -sos_docker_net: +{% for NETNAME, NETWORK in DOCKERMERGED.networks.items() %} +{% if not NETWORK.get('manager_only') or GLOBALS.get('is_manager', False) %} +sos_docker_net_{{ NETNAME }}: docker_network.present: - - name: sobridge - - subnet: {{ DOCKERMERGED.range }} - - gateway: {{ DOCKERMERGED.gateway }} + - name: {{ NETNAME }} + - subnet: {{ NETWORK.range }} + - gateway: {{ NETWORK.gateway }} - options: - com.docker.network.bridge.name: 'sobridge' + com.docker.network.bridge.name: '{{ NETNAME }}' com.docker.network.driver.mtu: '1500' com.docker.network.bridge.enable_ip_masquerade: 'true' com.docker.network.bridge.enable_icc: 'true' com.docker.network.bridge.host_binding_ipv4: '0.0.0.0' - - unless: ip l | grep sobridge + - unless: ip l | grep {{ NETNAME }} +{% endif %} +{% endfor %} diff --git a/salt/docker/soc_docker.yaml b/salt/docker/soc_docker.yaml index e649700da..30ad1e17b 100644 --- a/salt/docker/soc_docker.yaml +++ b/salt/docker/soc_docker.yaml @@ -7,6 +7,16 @@ docker: description: Default docker IP range for containers. helpLink: docker advanced: True + networks: + description: | + Docker networks used by the grid. sobridge carries most containers and takes its range and + gateway from the docker.range and docker.gateway settings above. soauth is an isolated + network for the authentication services, so that the Kratos admin API is only reachable + from the containers placed on it. Changing these requires a corresponding firewall rebuild. + helpLink: docker + readonly: True + advanced: True + global: True ulimits: description: | Default ulimit settings applied to all containers via the Docker daemon. Each entry specifies a resource name (e.g. nofile, memlock, core, nproc) with soft and hard limits. Individual container ulimits override these defaults. Valid resource names include: cpu, fsize, data, stack, core, rss, nproc, nofile, memlock, as, locks, sigpending, msgqueue, nice, rtprio, rttime. @@ -34,6 +44,16 @@ docker: readonly: True advanced: True global: True + networks: + description: | + Docker networks this container is attached to. The first entry is the container's + primary network and determines the address its published ports are forwarded to. + Defaults to sobridge when unset. + helpLink: docker + readonly: True + advanced: True + global: True + forcedType: "[]string" port_bindings: description: List of port bindings for the container. helpLink: docker diff --git a/salt/firewall/iptables.jinja b/salt/firewall/iptables.jinja index 91cfd92ec..a55486c46 100644 --- a/salt/firewall/iptables.jinja +++ b/salt/firewall/iptables.jinja @@ -4,11 +4,19 @@ {%- set role = GLOBALS.role.split('-')[1] %} {%- from 'firewall/containers.map.jinja' import NODE_CONTAINERS %} +{%- set NODE_NETWORKS = [] %} +{%- for NETNAME, NETWORK in DOCKERMERGED.networks.items() %} +{%- if not NETWORK.get('manager_only') or GLOBALS.get('is_manager', False) %} +{%- do NODE_NETWORKS.append(NETNAME) %} +{%- endif %} +{%- endfor %} + {%- set PR = [] %} {%- set D1 = [] %} {%- set D2 = [] %} {%- for container in NODE_CONTAINERS %} {%- set IP = DOCKERMERGED.containers[container].ip %} +{%- set BRIDGE = DOCKERMERGED.containers[container].network %} {%- if DOCKERMERGED.containers[container].port_bindings is defined %} {%- for binding in DOCKERMERGED.containers[container].port_bindings %} {#- cant split int so we convert to string #} @@ -35,11 +43,11 @@ {%- endif %} {%- do PR.append("-A POSTROUTING -s " ~ DOCKERMERGED.containers[container].ip ~ "/32 -d " ~ DOCKERMERGED.containers[container].ip ~ "/32 -p " ~ proto ~ " -m " ~ proto ~ " --dport " ~ containerPort ~ " -j MASQUERADE") %} {%- if bindip | length and bindip != '0.0.0.0' %} -{%- do D1.append("-A DOCKER -d " ~ bindip ~ "/32 ! -i sobridge -p " ~ proto ~ " -m " ~ proto ~ " --dport " ~ hostPort ~ " -j DNAT --to-destination " ~ DOCKERMERGED.containers[container].ip ~ ":" ~ containerPort) %} +{%- do D1.append("-A DOCKER -d " ~ bindip ~ "/32 ! -i " ~ BRIDGE ~ " -p " ~ proto ~ " -m " ~ proto ~ " --dport " ~ hostPort ~ " -j DNAT --to-destination " ~ DOCKERMERGED.containers[container].ip ~ ":" ~ containerPort) %} {%- else %} -{%- do D1.append("-A DOCKER ! -i sobridge -p " ~ proto ~ " -m " ~ proto ~ " --dport " ~ hostPort ~ " -j DNAT --to-destination " ~ DOCKERMERGED.containers[container].ip ~ ":" ~ containerPort) %} +{%- do D1.append("-A DOCKER ! -i " ~ BRIDGE ~ " -p " ~ proto ~ " -m " ~ proto ~ " --dport " ~ hostPort ~ " -j DNAT --to-destination " ~ DOCKERMERGED.containers[container].ip ~ ":" ~ containerPort) %} {%- endif %} -{%- do D2.append("-A DOCKER -d " ~ DOCKERMERGED.containers[container].ip ~ "/32 ! -i sobridge -o sobridge -p " ~ proto ~ " -m " ~ proto ~ " --dport " ~ containerPort ~ " -j ACCEPT") %} +{%- do D2.append("-A DOCKER -d " ~ DOCKERMERGED.containers[container].ip ~ "/32 ! -i " ~ BRIDGE ~ " -o " ~ BRIDGE ~ " -p " ~ proto ~ " -m " ~ proto ~ " --dport " ~ containerPort ~ " -j ACCEPT") %} {%- endfor %} {%- endif %} {%- endfor %} @@ -52,11 +60,15 @@ :DOCKER - [0:0] -A PREROUTING -m addrtype --dst-type LOCAL -j DOCKER -A OUTPUT ! -d 127.0.0.0/8 -m addrtype --dst-type LOCAL -j DOCKER --A POSTROUTING -s {{DOCKERMERGED.range}} ! -o sobridge -j MASQUERADE +{%- for NETNAME in NODE_NETWORKS %} +-A POSTROUTING -s {{ DOCKERMERGED.networks[NETNAME].range }} ! -o {{ NETNAME }} -j MASQUERADE +{%- endfor %} {%- for rule in PR %} {{ rule }} {%- endfor %} --A DOCKER -i sobridge -j RETURN +{%- for NETNAME in NODE_NETWORKS %} +-A DOCKER -i {{ NETNAME }} -j RETURN +{%- endfor %} {%- for rule in D1 %} {{ rule }} {%- endfor %} @@ -97,10 +109,12 @@ COMMIT {%- endif %} -A FORWARD -j DOCKER-USER -A FORWARD -j DOCKER-ISOLATION-STAGE-1 --A FORWARD -o sobridge -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT --A FORWARD -o sobridge -j DOCKER --A FORWARD -i sobridge ! -o sobridge -j ACCEPT --A FORWARD -i sobridge -o sobridge -j ACCEPT +{%- for NETNAME in NODE_NETWORKS %} +-A FORWARD -o {{ NETNAME }} -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +-A FORWARD -o {{ NETNAME }} -j DOCKER +-A FORWARD -i {{ NETNAME }} ! -o {{ NETNAME }} -j ACCEPT +-A FORWARD -i {{ NETNAME }} -o {{ NETNAME }} -j ACCEPT +{%- endfor %} -A FORWARD -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT -A FORWARD -i lo -j ACCEPT -A FORWARD -m conntrack --ctstate INVALID -j DROP @@ -112,13 +126,18 @@ COMMIT {%- for rule in D2 %} {{ rule }} {%- endfor %} - --A DOCKER-ISOLATION-STAGE-1 -i sobridge ! -o sobridge -j DOCKER-ISOLATION-STAGE-2 +{% for NETNAME in NODE_NETWORKS %} +-A DOCKER-ISOLATION-STAGE-1 -i {{ NETNAME }} ! -o {{ NETNAME }} -j DOCKER-ISOLATION-STAGE-2 +{%- endfor %} -A DOCKER-ISOLATION-STAGE-1 -j RETURN --A DOCKER-ISOLATION-STAGE-2 -o sobridge -j DROP +{%- for NETNAME in NODE_NETWORKS %} +-A DOCKER-ISOLATION-STAGE-2 -o {{ NETNAME }} -j DROP +{%- endfor %} -A DOCKER-ISOLATION-STAGE-2 -j RETURN --A DOCKER-USER ! -i sobridge -o sobridge -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT --A DOCKER-USER ! -i sobridge -o sobridge -j LOGGING +{%- for NETNAME in NODE_NETWORKS %} +-A DOCKER-USER ! -i {{ NETNAME }} -o {{ NETNAME }} -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +-A DOCKER-USER ! -i {{ NETNAME }} -o {{ NETNAME }} -j LOGGING +{%- endfor %} -A DOCKER-USER -j RETURN -A LOGGING -m limit --limit 2/min -j LOG --log-prefix "IPTables-dropped: " -A LOGGING -j DROP diff --git a/salt/firewall/map.jinja b/salt/firewall/map.jinja index 58d8c189d..93af5cace 100644 --- a/salt/firewall/map.jinja +++ b/salt/firewall/map.jinja @@ -4,8 +4,12 @@ {# add our ip to self #} {% do FIREWALL_DEFAULT.firewall.hostgroups.self.append(GLOBALS.node_ip) %} -{# add dockernet range #} -{% do FIREWALL_DEFAULT.firewall.hostgroups.dockernet.append(DOCKERMERGED.range) %} +{# add dockernet ranges #} +{% for NETNAME, NETWORK in DOCKERMERGED.networks.items() %} +{% if not NETWORK.get('manager_only') or GLOBALS.get('is_manager', False) %} +{% do FIREWALL_DEFAULT.firewall.hostgroups.dockernet.append(NETWORK.range) %} +{% endif %} +{% endfor %} {% if GLOBALS.role == 'so-idh' %} {% from 'idh/opencanary_config.map.jinja' import IDH_PORTGROUPS %} diff --git a/salt/kratos/enabled.sls b/salt/kratos/enabled.sls index ab13d759f..38a710cfc 100644 --- a/salt/kratos/enabled.sls +++ b/salt/kratos/enabled.sls @@ -19,8 +19,8 @@ so-kratos: - hostname: kratos - name: so-kratos - networks: - - sobridge: - - ipv4_address: {{ DOCKERMERGED.containers['so-kratos'].ip }} + - soauth: + - ipv4_address: {{ DOCKERMERGED.containers['so-kratos'].ips['soauth'] }} - binds: - /opt/so/conf/kratos/:/kratos-conf:ro - /opt/so/log/kratos/:/kratos-log:rw @@ -71,7 +71,7 @@ delete_so-kratos_so-status.disabled: wait_for_kratos: http.wait_for_successful_query: - - name: 'http://{{ GLOBALS.manager }}:4434/' + - name: 'http://{{ DOCKERMERGED.containers['so-kratos'].ips['soauth'] }}:4434/' - ssl: True - verify_ssl: False - status: diff --git a/salt/manager/tools/sbin/so-user b/salt/manager/tools/sbin/so-user index 060dcf3a4..e5537173f 100755 --- a/salt/manager/tools/sbin/so-user +++ b/salt/manager/tools/sbin/so-user @@ -129,7 +129,8 @@ while [[ $# -gt 0 ]]; do esac done -kratosUrl=${KRATOS_URL:-http://127.0.0.1:4434/admin} +kratosContainer=${KRATOS_CONTAINER:-so-kratos} +kratosUrl=${KRATOS_URL:-http://localhost:4434/admin} databasePath=${KRATOS_DB_PATH:-/nsm/kratos/db/db.sqlite} databaseTimeout=${KRATOS_DB_TIMEOUT:-5000} bcryptRounds=${BCRYPT_ROUNDS:-12} @@ -154,6 +155,10 @@ function fail() { exit 1 } +function kratosCurl() { + docker exec -i "$kratosContainer" curl "$@" +} + function require() { cmd=$1 which "$1" 2>&1 > /dev/null @@ -164,18 +169,18 @@ function require() { function verifyEnvironment() { require "htpasswd" require "jq" - require "curl" + require "docker" require "openssl" require "sqlite3" [[ ! -f $databasePath ]] && fail "Unable to find database file; specify path via KRATOS_DB_PATH environment variable" - response=$(curl -Ss -L ${kratosUrl}/) + response=$(kratosCurl -Ss -L ${kratosUrl}/) [[ "$response" != "404 page not found" ]] && fail "Unable to communicate with Kratos; specify URL via KRATOS_URL environment variable" } function findIdByEmail() { email=${1,,} - response=$(curl -Ss -L ${kratosUrl}/identities) + response=$(kratosCurl -Ss -L ${kratosUrl}/identities) identityId=$(echo "${response}" | jq -r ".[] | select(.verifiable_addresses[0].value == \"$email\") | .id") echo $identityId } @@ -416,7 +421,7 @@ function syncAll() { } function listUsers() { - response=$(curl -Ss -L ${kratosUrl}/identities) + response=$(kratosCurl -Ss -L ${kratosUrl}/identities) [[ $? != 0 ]] && fail "Unable to communicate with Kratos" users=$(echo "${response}" | jq -r ".[] | .verifiable_addresses[0].value" | sort) @@ -495,7 +500,7 @@ function createUser() { EOF ) - response=$(curl -Ss -L ${kratosUrl}/identities -d "$addUserJson") + response=$(kratosCurl -Ss -L ${kratosUrl}/identities -d "$addUserJson") [[ $? != 0 ]] && fail "Unable to communicate with Kratos" identityId=$(echo "${response}" | jq -r ".id") @@ -518,7 +523,7 @@ function updateStatus() { identityId=$(findIdByEmail "$email") [[ ${identityId} == "" ]] && fail "User not found" - response=$(curl -Ss -L "${kratosUrl}/identities/$identityId") + response=$(kratosCurl -Ss -L "${kratosUrl}/identities/$identityId") [[ $? != 0 ]] && fail "Unable to communicate with Kratos" schemaId=$(echo "$response" | jq -r .schema_id) @@ -531,7 +536,7 @@ function updateStatus() { state="inactive" fi body="{ \"schema_id\": \"$schemaId\", \"state\": \"$state\", \"traits\": $traitBlock }" - response=$(curl -fSsL -XPUT -H "Content-Type: application/json" "${kratosUrl}/identities/$identityId" -d "$body") + response=$(kratosCurl -fSsL -XPUT -H "Content-Type: application/json" "${kratosUrl}/identities/$identityId" -d "$body") [[ $? != 0 ]] && fail "Unable to update user" } @@ -550,7 +555,7 @@ function updateUserProfile() { identityId=$(findIdByEmail "$email") [[ ${identityId} == "" ]] && fail "User not found" - response=$(curl -Ss -L "${kratosUrl}/identities/$identityId") + response=$(kratosCurl -Ss -L "${kratosUrl}/identities/$identityId") [[ $? != 0 ]] && fail "Unable to communicate with Kratos" schemaId=$(echo "$response" | jq -r .schema_id) @@ -559,7 +564,7 @@ function updateUserProfile() { traitBlock="{\"email\":\"$email\",\"firstName\":\"$firstName\",\"lastName\":\"$lastName\",\"note\":\"$note\"}" body="{ \"schema_id\": \"$schemaId\", \"state\": \"$state\", \"traits\": $traitBlock }" - response=$(curl -fSsL -XPUT -H "Content-Type: application/json" "${kratosUrl}/identities/$identityId" -d "$body") + response=$(kratosCurl -fSsL -XPUT -H "Content-Type: application/json" "${kratosUrl}/identities/$identityId" -d "$body") [[ $? != 0 ]] && fail "Unable to update user" } @@ -569,7 +574,7 @@ function deleteUser() { identityId=$(findIdByEmail "$email") [[ ${identityId} == "" ]] && fail "User not found" - response=$(curl -Ss -XDELETE -L "${kratosUrl}/identities/$identityId") + response=$(kratosCurl -Ss -XDELETE -L "${kratosUrl}/identities/$identityId") [[ $? != 0 ]] && fail "Unable to communicate with Kratos" rolesTmpFile="${socRolesFile}.tmp" diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 353158b7e..2bc1ea9e8 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -28,6 +28,7 @@ INSTALLEDSALTVERSION=$(salt --versions-report | grep Salt: | awk '{print $2}') # percentage like "25%"). Empty means so-soup-grid-highstate uses the salt:auto_apply:batch # pillar default. BATCHSIZE= +DEFAULT_DOCKER_RANGE='172.17.1.0/24' SOUP_LOG=/root/soup.log SOUP_DEBUG_LOG=/root/soup-debug.log WHATWOULDYOUSAYYAHDOHERE=soup @@ -525,6 +526,7 @@ preupgrade_changes() { [[ "$INSTALLEDVERSION" == "3.0.0" ]] && up_to_3.1.0 [[ "$INSTALLEDVERSION" == "3.1.0" ]] && up_to_3.2.0 [[ "$INSTALLEDVERSION" == "3.2.0" ]] && up_to_3.3.0 + [[ "$INSTALLEDVERSION" == "3.3.0" ]] && up_to_3.4.0 true } @@ -543,6 +545,7 @@ postupgrade_changes() { [[ "$POSTVERSION" == "3.0.0" ]] && post_to_3.1.0 [[ "$POSTVERSION" == "3.1.0" ]] && post_to_3.2.0 [[ "$POSTVERSION" == "3.2.0" ]] && post_to_3.3.0 + [[ "$POSTVERSION" == "3.3.0" ]] && post_to_3.4.0 # All applicable post-upgrade steps completed; clear the resume marker. rm -f "$POSTVERSION_FILE" true @@ -1093,6 +1096,82 @@ post_to_3.3.0() { } ### 3.3.0 End ### +### 3.4.0 Scripts ### +up_to_3.4.0() { + set_soauth_range + + echo "Removing so-kratos and so-soc so they are recreated on the soauth network." + docker rm -f so-kratos so-soc >> $SOUP_LOG 2>&1 + + INSTALLEDVERSION=3.4.0 +} + +set_soauth_range() { + local pillar_file=/opt/so/saltstack/local/pillar/docker/soc_docker.sls + local current_range suggested authnet authgw input + + [[ -f "$pillar_file" ]] || return 0 + + current_range=$(so-yaml.py get -r "$pillar_file" docker.range 2>/dev/null) || return 0 + + # A default range gets the 172.17.2.0/24 from docker/defaults.yaml, same as a fresh + # install, so there is nothing to ask about. + [[ -n "$current_range" && "$current_range" != "$DEFAULT_DOCKER_RANGE" ]] || return 0 + + if so-yaml.py get -r "$pillar_file" docker.networks.soauth.range >/dev/null 2>&1; then + return 0 + fi + + suggested=$(echo "${current_range%%/*}" | awk -F'.' '{ printf "%s.%s.%s.%s", $1, $2, ($3 + 1) % 256, $4 }') + + if [[ -z $UNATTENDED ]]; then + echo "" + echo "This grid uses a custom Docker range ($current_range). The authentication" + echo "services are moving to their own isolated network, which needs a second /24" + echo "that does not overlap it." + echo "" + while :; do + read -rp "Enter the network without the /24 suffix, or press Enter for ${suggested}: " input + [[ -z "$input" ]] && input="$suggested" + if valid_soauth_range "$input" "$current_range"; then + authnet="$input" + break + fi + echo "That range must be a valid IPv4 network, must not be within 172.17.0.0/24, and must not overlap ${current_range}." + done + else + if ! valid_soauth_range "$suggested" "$current_range"; then + FINAL_MESSAGE_QUEUE+=("WARNING: Unable to pick a range for the authentication network alongside $current_range. Set it manually before the next highstate:") + FINAL_MESSAGE_QUEUE+=(" - so-yaml.py add $pillar_file docker.networks.soauth.range /24") + FINAL_MESSAGE_QUEUE+=(" - so-yaml.py add $pillar_file docker.networks.soauth.gateway ") + return 0 + fi + authnet="$suggested" + FINAL_MESSAGE_QUEUE+=("NOTE: The authentication services moved to an isolated Docker network and were assigned ${authnet}/24.") + FINAL_MESSAGE_QUEUE+=(" - If that conflicts with your environment, update docker.networks.soauth in $pillar_file and run so-checkin.") + fi + + authgw=$(echo "$authnet" | awk -F'.' '{print $1,$2,$3,1}' OFS='.') + + echo "Assigning the authentication network the range ${authnet}/24." + so-yaml.py add "$pillar_file" docker.networks.soauth.range "${authnet}/24" >> $SOUP_LOG 2>&1 + so-yaml.py add "$pillar_file" docker.networks.soauth.gateway "$authgw" >> $SOUP_LOG 2>&1 +} + +valid_soauth_range() { + local candidate=$1 docker_range=$2 + + valid_ip4 "$candidate" || return 1 + [[ $candidate =~ ^172\.17\.0\. ]] && return 1 + [[ "${candidate}/24" == "$docker_range" ]] && return 1 + return 0 +} + +post_to_3.4.0() { + set_postversion 3.4.0 +} +### 3.4.0 End ### + repo_sync() { echo "Sync the local repo." diff --git a/salt/soc/defaults.map.jinja b/salt/soc/defaults.map.jinja index 93fb63efe..1271bc06e 100644 --- a/salt/soc/defaults.map.jinja +++ b/salt/soc/defaults.map.jinja @@ -14,6 +14,8 @@ {% do SOCDEFAULTS.soc.config.server.modules[module].update({'hostUrl': application_url}) %} {% endfor %} +{% do SOCDEFAULTS.soc.config.server.modules.kratos.update({'publicHostUrl': 'http://' ~ DOCKERMERGED.containers['so-kratos'].ips['soauth'] ~ ':4433/'}) %} + {# add all grid heavy nodes to soc.server.modules.elastic.remoteHostUrls #} {% for node_type, minions in salt['pillar.get']('elasticsearch:nodes', {}).items() %} {% if node_type in ['heavynode'] %} diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index d14ca3e08..19192dc22 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1380,6 +1380,7 @@ soc: retryFailureMaxAttempts: 5 kratos: hostUrl: + publicHostUrl: hydra: hostUrl: elastalertengine: diff --git a/salt/soc/enabled.sls b/salt/soc/enabled.sls index 5b49ed47e..945fe2254 100644 --- a/salt/soc/enabled.sls +++ b/salt/soc/enabled.sls @@ -23,7 +23,9 @@ so-soc: - name: so-soc - networks: - sobridge: - - ipv4_address: {{ DOCKERMERGED.containers['so-soc'].ip }} + - ipv4_address: {{ DOCKERMERGED.containers['so-soc'].ips['sobridge'] }} + - soauth: + - ipv4_address: {{ DOCKERMERGED.containers['so-soc'].ips['soauth'] }} - binds: - /nsm/rules:/nsm/rules:rw - /opt/so/conf/strelka:/opt/sensoroni/yara:rw diff --git a/salt/vars/globals.map.jinja b/salt/vars/globals.map.jinja index 385db02ae..772fe8797 100644 --- a/salt/vars/globals.map.jinja +++ b/salt/vars/globals.map.jinja @@ -55,7 +55,7 @@ do GLOBALS.update({ 'application_urls': { 'hydra': 'http://' ~ GLOBALS.manager ~ ':4445/', - 'kratos': 'http://' ~ GLOBALS.manager ~ ':4434/', + 'kratos': 'http://' ~ DOCKERMERGED.containers['so-kratos'].ips['soauth'] ~ ':4434/', 'elastic': 'https://' ~ GLOBALS.manager ~ ':9200/', 'influxdb': 'https://' ~ GLOBALS.manager ~ ':8086/' } diff --git a/setup/so-functions b/setup/so-functions index 1269b4e9d..8ece2d9a2 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -276,9 +276,20 @@ collect_dockernet() { whiptail_invalid_input whiptail_dockernet_sosnet "$DOCKERNET" done + + whiptail_authnet_sosnet "$(adjacent_net "$DOCKERNET")" + + while ! valid_ip4 "$AUTHNET" || [[ $AUTHNET =~ "172.17.0." ]] || [[ "$AUTHNET" == "$DOCKERNET" ]]; do + whiptail_invalid_input + whiptail_authnet_sosnet "$AUTHNET" + done fi } +adjacent_net() { + echo "$1" | awk -F'.' '{ printf "%s.%s.%s.%s", $1, $2, ($3 + 1) % 256, $4 }' +} + collect_gateway() { whiptail_management_interface_gateway @@ -1399,6 +1410,15 @@ docker_pillar() { "docker:"\ " range: '$DOCKERNET/24'"\ " gateway: '$DOCKERGATEWAY'" > $docker_pillar_file + + if [ ! -z "$AUTHNET" ]; then + AUTHGATEWAY=$(echo $AUTHNET | awk -F'.' '{print $1,$2,$3,1}' OFS='.') + printf '%s\n'\ + " networks:"\ + " soauth:"\ + " range: '$AUTHNET/24'"\ + " gateway: '$AUTHGATEWAY'" >> $docker_pillar_file + fi fi } diff --git a/setup/so-whiptail b/setup/so-whiptail index 6188d3d30..0d321bc19 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -365,6 +365,18 @@ whiptail_dockernet_sosnet() { } +whiptail_authnet_sosnet() { + + [ -n "$TESTING" ] && return + + AUTHNET=$(whiptail --title "$whiptail_title" --inputbox \ + "\nEnter a second /24 size network range WITHOUT the /24 suffix. The authentication services are isolated on their own network so that the identity provider is not reachable from other containers. It must not overlap the range you just entered, and any range within 172.17.0.0/24 cannot be used." 13 65 "$1" 3>&1 1>&2 2>&3) + + local exitstatus=$? + whiptail_check_exitstatus $exitstatus + +} + whiptail_end_settings() { [ -n "$TESTING" ] && return @@ -427,6 +439,7 @@ whiptail_end_settings() { [[ -n $WEBUSER ]] && __append_end_msg "Web User: $WEBUSER" [[ -n $DOCKERNET ]] && __append_end_msg "Docker network: $DOCKERNET/24" + [[ -n $AUTHNET ]] && __append_end_msg "Authentication network: $AUTHNET/24" if [[ ${#ntp_servers[@]} -gt 0 ]]; then __append_end_msg "NTP Servers:" for server in "${ntp_servers[@]}"; do From bbc28c88b713fccfa762b2e7aabf93387bda7da4 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 17 Sep 2026 16:58:49 -0400 Subject: [PATCH 2/3] Isolate the Hydra admin API on the soauth docker network The Hydra admin API creates OAuth clients and introspects tokens without authentication, relying on network isolation the same way Kratos does, but so-hydra published 0.0.0.0:4445:4445 and sat on sobridge. With userland-proxy left at its default of true, Docker ran a proxy listener on the host, so any container reached the admin API through the manager IP or the bridge gateway regardless of its docker network. so-hydra now sits alone on soauth and no longer publishes 4445. 4444 is still published, so the nginx proxy for /oauth2/token and the well-known endpoints is unchanged. so-soc is already dual homed from the Kratos change and reaches the admin API over soauth, so only its hostUrl moves. wait_for_hydra polls the container address instead of the host port. so-client reaches the admin API through docker exec, mirroring so-user. Exit codes still propagate, so the --fail-with-body error handling is unchanged. so-hydra is removed in up_to_3.4.0 alongside so-kratos and so-soc so the highstate recreates it with the correct network membership. The auth network range is already handled by set_soauth_range. --- salt/docker/defaults.yaml | 2 +- salt/hydra/enabled.sls | 6 +++--- salt/manager/tools/sbin/so-client | 21 +++++++++++++-------- salt/manager/tools/sbin/soup | 4 ++-- salt/vars/globals.map.jinja | 2 +- 5 files changed, 20 insertions(+), 15 deletions(-) diff --git a/salt/docker/defaults.yaml b/salt/docker/defaults.yaml index ea656f3dd..b1e0d9162 100644 --- a/salt/docker/defaults.yaml +++ b/salt/docker/defaults.yaml @@ -73,9 +73,9 @@ docker: ulimits: [] 'so-hydra': final_octet: 30 + networks: ['soauth'] port_bindings: - 0.0.0.0:4444:4444 - - 0.0.0.0:4445:4445 custom_bind_mounts: [] extra_hosts: [] extra_env: [] diff --git a/salt/hydra/enabled.sls b/salt/hydra/enabled.sls index 74106f550..13fce79cf 100644 --- a/salt/hydra/enabled.sls +++ b/salt/hydra/enabled.sls @@ -26,8 +26,8 @@ so-hydra: - hostname: hydra - name: so-hydra - networks: - - sobridge: - - ipv4_address: {{ DOCKERMERGED.containers['so-hydra'].ip }} + - soauth: + - ipv4_address: {{ DOCKERMERGED.containers['so-hydra'].ips['soauth'] }} - binds: - /opt/so/conf/hydra/:/hydra-conf:ro - /opt/so/log/hydra/:/hydra-log:rw @@ -73,7 +73,7 @@ delete_so-hydra_so-status.disabled: wait_for_hydra: http.wait_for_successful_query: - - name: 'http://{{ GLOBALS.manager }}:4444/health/alive' + - name: 'http://{{ DOCKERMERGED.containers['so-hydra'].ips['soauth'] }}:4444/health/alive' - ssl: True - verify_ssl: False - status: diff --git a/salt/manager/tools/sbin/so-client b/salt/manager/tools/sbin/so-client index f979d7bc0..0bafde603 100755 --- a/salt/manager/tools/sbin/so-client +++ b/salt/manager/tools/sbin/so-client @@ -106,7 +106,8 @@ while [[ $# -gt 0 ]]; do esac done -hydraUrl=${HYDRA_URL:-http://127.0.0.1:4445} +hydraContainer=${HYDRA_CONTAINER:-so-hydra} +hydraUrl=${HYDRA_URL:-http://localhost:4445} socRolesFile=${SOC_ROLES_FILE:-/opt/so/conf/soc/soc_clients_roles} soUID=${SOCORE_UID:-939} soGID=${SOCORE_GID:-939} @@ -124,6 +125,10 @@ function fail() { exit 1 } +function hydraCurl() { + docker exec -i "$hydraContainer" curl "$@" +} + function require() { cmd=$1 which "$1" 2>&1 > /dev/null @@ -133,8 +138,8 @@ function require() { # Verify this environment is capable of running this script function verifyEnvironment() { require "jq" - require "curl" - response=$(curl -Ss -L ${hydraUrl}/health/alive) + require "docker" + response=$(hydraCurl -Ss -L ${hydraUrl}/health/alive) [[ "$response" != '{"status":"ok"}' ]] && fail "Unable to communicate with Hydra; specify URL via HYDRA_URL environment variable" } @@ -164,7 +169,7 @@ function ensureRoleFileExists() { } function listClients() { - response=$(curl -Ss -L -f ${hydraUrl}/admin/clients) + response=$(hydraCurl -Ss -L -f ${hydraUrl}/admin/clients) [[ $? != 0 ]] && fail "Unable to communicate with Hydra" clientIds=$(echo "${response}" | jq -r ".[] | .client_id" | sort) @@ -251,7 +256,7 @@ function createClient() { EOF ) - response=$(curl -Ss -L --fail-with-body -X POST ${hydraUrl}/admin/clients -d "$body") + response=$(hydraCurl -Ss -L --fail-with-body -X POST ${hydraUrl}/admin/clients -d "$body") if [[ $? != 0 ]]; then error=$(echo $response | jq .error) fail "Failed to submit request to Hydra: $error" @@ -283,7 +288,7 @@ function update() { EOF ) - response=$(curl -Ss -L --fail-with-body -X PATCH ${hydraUrl}/admin/clients/$id -d "$body") + response=$(hydraCurl -Ss -L --fail-with-body -X PATCH ${hydraUrl}/admin/clients/$id -d "$body") if [[ $? != 0 ]]; then error=$(echo $response | jq .error) fail "Failed to submit request to Hydra: $error" @@ -305,7 +310,7 @@ function generateSecret() { EOF ) - response=$(curl -Ss -L --fail-with-body -X PATCH ${hydraUrl}/admin/clients/$id -d "$body") + response=$(hydraCurl -Ss -L --fail-with-body -X PATCH ${hydraUrl}/admin/clients/$id -d "$body") if [[ $? != 0 ]]; then error=$(echo $response | jq .error) fail "Failed to submit request to Hydra: $error" @@ -317,7 +322,7 @@ function deleteClient() { [[ ${identityId} == "" ]] && fail "Client not found" - response=$(curl -Ss -XDELETE -L --fail-with-body "${hydraUrl}/admin/clients/$identityId") + response=$(hydraCurl -Ss -XDELETE -L --fail-with-body "${hydraUrl}/admin/clients/$identityId") if [[ $? != 0 ]]; then error=$(echo $response | jq .error) fail "Failed to submit request to Hydra: $error" diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 2bc1ea9e8..62bc7d266 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -1100,8 +1100,8 @@ post_to_3.3.0() { up_to_3.4.0() { set_soauth_range - echo "Removing so-kratos and so-soc so they are recreated on the soauth network." - docker rm -f so-kratos so-soc >> $SOUP_LOG 2>&1 + echo "Removing so-kratos, so-hydra and so-soc so they are recreated on the soauth network." + docker rm -f so-kratos so-hydra so-soc >> $SOUP_LOG 2>&1 INSTALLEDVERSION=3.4.0 } diff --git a/salt/vars/globals.map.jinja b/salt/vars/globals.map.jinja index 772fe8797..5060806d5 100644 --- a/salt/vars/globals.map.jinja +++ b/salt/vars/globals.map.jinja @@ -54,7 +54,7 @@ {% do GLOBALS.update({ 'application_urls': { - 'hydra': 'http://' ~ GLOBALS.manager ~ ':4445/', + 'hydra': 'http://' ~ DOCKERMERGED.containers['so-hydra'].ips['soauth'] ~ ':4445/', 'kratos': 'http://' ~ DOCKERMERGED.containers['so-kratos'].ips['soauth'] ~ ':4434/', 'elastic': 'https://' ~ GLOBALS.manager ~ ':9200/', 'influxdb': 'https://' ~ GLOBALS.manager ~ ':8086/' From 65e261475d4d35fd52ed01ac6111dd900c69c5ac Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 18 Sep 2026 10:17:20 -0400 Subject: [PATCH 3/3] Mention Hydra in the soauth network description The description was written alongside the Kratos change and named only the Kratos admin API, but so-hydra now sits on soauth too. The trailing note about rebuilding the firewall is dropped, since the setting is readonly and cannot be changed from SOC in the first place. --- salt/docker/soc_docker.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/docker/soc_docker.yaml b/salt/docker/soc_docker.yaml index 30ad1e17b..18a7278a5 100644 --- a/salt/docker/soc_docker.yaml +++ b/salt/docker/soc_docker.yaml @@ -11,8 +11,8 @@ docker: description: | Docker networks used by the grid. sobridge carries most containers and takes its range and gateway from the docker.range and docker.gateway settings above. soauth is an isolated - network for the authentication services, so that the Kratos admin API is only reachable - from the containers placed on it. Changing these requires a corresponding firewall rebuild. + network for the authentication services, so that the Kratos admin and Hydra API is only reachable + from the containers placed on it. helpLink: docker readonly: True advanced: True