Merge remote-tracking branch 'origin/3/dev' into zeekrestart

filterEventFields

salt/docker/defaults.yaml

@@ -1,12 +1,6 @@
 docker:
   range: '172.17.1.0/24'
   gateway: '172.17.1.1'
-  networks:
-    sobridge: {}
-    soauth:
-      range: '172.17.2.0/24'
-      gateway: '172.17.2.1'
-      manager_only: True
   ulimits:
     - name: nofile
       soft: 1048576
@@ -64,18 +58,18 @@ docker:
       ulimits: []
     'so-kratos':
       final_octet: 28
-      networks: ['soauth']
       port_bindings:
         - 0.0.0.0:4433:4433
+        - 0.0.0.0:4434:4434
       custom_bind_mounts: []
       extra_hosts: []
       extra_env: []
       ulimits: []
     'so-hydra':
       final_octet: 30
-      networks: ['soauth']
       port_bindings:
         - 0.0.0.0:4444:4444
+        - 0.0.0.0:4445:4445
       custom_bind_mounts: []
       extra_hosts: []
       extra_env: []
@@ -134,7 +128,6 @@ docker:
       ulimits: []
     'so-soc':
       final_octet: 34
-      networks: ['sobridge', 'soauth']
       port_bindings:
         - 0.0.0.0:9822:9822
       custom_bind_mounts: []

salt/docker/docker.map.jinja

@@ -1,26 +1,8 @@
 {% import_yaml 'docker/defaults.yaml' as DOCKERDEFAULTS %}
 {% set DOCKERMERGED = salt['pillar.get']('docker', DOCKERDEFAULTS.docker, merge=True) %}
-
-{% if DOCKERMERGED.networks.sobridge is not mapping %}
-{%   do DOCKERMERGED.networks.update({'sobridge': {}}) %}
-{% endif %}
-{% do DOCKERMERGED.networks['sobridge'].update({'range': DOCKERMERGED.range, 'gateway': DOCKERMERGED.gateway}) %}
-
-{% for netname, net in DOCKERMERGED.networks.items() %}
-{%   set RANGESPLIT = net.range.split('.') %}
-{%   do net.update({'prefix': RANGESPLIT[0] ~ '.' ~ RANGESPLIT[1] ~ '.' ~ RANGESPLIT[2] ~ '.'}) %}
-{% endfor %}
+{% set RANGESPLIT = DOCKERMERGED.range.split('.') %}
+{% set FIRSTTHREE = RANGESPLIT[0] ~ '.' ~ RANGESPLIT[1] ~ '.' ~ RANGESPLIT[2] ~ '.' %}
 
 {% for container, vals in DOCKERMERGED.containers.items() %}
-{%   set CONTAINER_NETS = vals.get('networks', ['sobridge']) %}
-{%   set IPS = {} %}
-{%   for netname in CONTAINER_NETS %}
-{%     do IPS.update({netname: DOCKERMERGED.networks[netname].prefix ~ vals.final_octet}) %}
-{%   endfor %}
-{%   do DOCKERMERGED.containers[container].update({
-         'networks': CONTAINER_NETS,
-         'ips': IPS,
-         'network': CONTAINER_NETS[0],
-         'ip': IPS[CONTAINER_NETS[0]]
-       }) %}
+{%   do DOCKERMERGED.containers[container].update({'ip': FIRSTTHREE ~ DOCKERMERGED.containers[container].final_octet}) %}
 {% endfor %}

salt/docker/init.sls

@@ -71,19 +71,15 @@ dockerreserveports:
     - source: salt://common/files/99-reserved-ports.conf
     - name: /etc/sysctl.d/99-reserved-ports.conf
 
-{% for NETNAME, NETWORK in DOCKERMERGED.networks.items() %}
-{% if not NETWORK.get('manager_only') or GLOBALS.get('is_manager', False) %}
-sos_docker_net_{{ NETNAME }}:
+sos_docker_net:
   docker_network.present:
-    - name: {{ NETNAME }}
-    - subnet: {{ NETWORK.range }}
-    - gateway: {{ NETWORK.gateway }}
+    - name: sobridge
+    - subnet: {{ DOCKERMERGED.range }}
+    - gateway: {{ DOCKERMERGED.gateway }}
     - options:
-        com.docker.network.bridge.name: '{{ NETNAME }}'
+        com.docker.network.bridge.name: 'sobridge'
         com.docker.network.driver.mtu: '1500'
         com.docker.network.bridge.enable_ip_masquerade: 'true'
         com.docker.network.bridge.enable_icc: 'true'
         com.docker.network.bridge.host_binding_ipv4: '0.0.0.0'
-    - unless: ip l | grep {{ NETNAME }}
-{% endif %}
-{% endfor %}
+    - unless: ip l | grep sobridge

salt/docker/soc_docker.yaml

@@ -7,33 +7,6 @@ docker:
     description: Default docker IP range for containers.
     helpLink: docker
     advanced: True
-  networks:
-    soauth:
-      range:
-        description: |
-          IP range for the soauth docker network. soauth is an isolated network for the
-          authentication services, so that the Kratos and Hydra admin APIs are only reachable
-          from the containers placed on it. Most containers instead use sobridge, which takes
-          its range and gateway from the docker.range and docker.gateway settings above.
-        helpLink: docker
-        readonly: True
-        advanced: True
-        global: True
-      gateway:
-        description: Gateway for the soauth docker network.
-        helpLink: docker
-        readonly: True
-        advanced: True
-        global: True
-      manager_only:
-        description: |
-          Limits the soauth network to grid members running the authentication containers,
-          instead of creating it on every node.
-        helpLink: docker
-        readonly: True
-        advanced: True
-        global: True
-        forcedType: bool
   ulimits:
     description: |
       Default ulimit settings applied to all containers via the Docker daemon. Each entry specifies a resource name (e.g. nofile, memlock, core, nproc) with soft and hard limits. Individual container ulimits override these defaults. Valid resource names include: cpu, fsize, data, stack, core, rss, nproc, nofile, memlock, as, locks, sigpending, msgqueue, nice, rtprio, rttime.
@@ -61,16 +34,6 @@ docker:
         readonly: True
         advanced: True
         global: True
-      networks:
-        description: |
-          Docker networks this container is attached to. The first entry is the container's
-          primary network and determines the address its published ports are forwarded to.
-          Defaults to sobridge when unset.
-        helpLink: docker
-        readonly: True
-        advanced: True
-        global: True
-        forcedType: "[]string"
       port_bindings:
         description: List of port bindings for the container.
         helpLink: docker

salt/firewall/iptables.jinja

@@ -4,19 +4,11 @@
 {%- set role = GLOBALS.role.split('-')[1] %}
 {%- from 'firewall/containers.map.jinja' import NODE_CONTAINERS %}
 
-{%- set NODE_NETWORKS = [] %}
-{%- for NETNAME, NETWORK in DOCKERMERGED.networks.items() %}
-{%-   if not NETWORK.get('manager_only') or GLOBALS.get('is_manager', False) %}
-{%-     do NODE_NETWORKS.append(NETNAME) %}
-{%-   endif %}
-{%- endfor %}
-
 {%- set PR = [] %}
 {%- set D1 = [] %}
 {%- set D2 = [] %}
 {%- for container in NODE_CONTAINERS %}
 {%-   set IP = DOCKERMERGED.containers[container].ip %}
-{%-   set BRIDGE = DOCKERMERGED.containers[container].network %}
 {%-   if DOCKERMERGED.containers[container].port_bindings is defined %}
 {%-     for binding in DOCKERMERGED.containers[container].port_bindings %}
 {#-       cant split int so we convert to string #}
@@ -43,11 +35,11 @@
 {%-         endif %}
 {%-         do PR.append("-A POSTROUTING -s " ~ DOCKERMERGED.containers[container].ip ~ "/32 -d " ~ DOCKERMERGED.containers[container].ip ~ "/32 -p " ~  proto ~ " -m " ~ proto  ~ " --dport " ~ containerPort ~ " -j MASQUERADE") %}
 {%-         if bindip | length and bindip != '0.0.0.0' %}
-{%-           do D1.append("-A DOCKER -d " ~ bindip ~ "/32 ! -i " ~ BRIDGE ~ " -p " ~ proto ~ " -m " ~ proto ~ " --dport " ~ hostPort ~ " -j DNAT --to-destination " ~ DOCKERMERGED.containers[container].ip ~ ":" ~ containerPort) %}
+{%-           do D1.append("-A DOCKER -d " ~ bindip ~ "/32 ! -i sobridge -p " ~ proto ~ " -m " ~ proto ~ " --dport " ~ hostPort ~ " -j DNAT --to-destination " ~ DOCKERMERGED.containers[container].ip ~ ":" ~ containerPort) %}
 {%-         else %}
-{%-           do D1.append("-A DOCKER ! -i " ~ BRIDGE ~ " -p " ~ proto ~ " -m " ~ proto ~ " --dport " ~ hostPort ~ " -j DNAT --to-destination " ~ DOCKERMERGED.containers[container].ip ~ ":" ~ containerPort) %}
+{%-           do D1.append("-A DOCKER ! -i sobridge -p " ~ proto ~ " -m " ~ proto ~ " --dport " ~ hostPort ~ " -j DNAT --to-destination " ~ DOCKERMERGED.containers[container].ip ~ ":" ~ containerPort) %}
 {%-         endif %}
-{%-         do D2.append("-A DOCKER -d " ~ DOCKERMERGED.containers[container].ip ~ "/32 ! -i " ~ BRIDGE ~ " -o " ~ BRIDGE ~ " -p " ~ proto ~ " -m " ~ proto ~ " --dport " ~ containerPort ~ " -j ACCEPT") %}
+{%-         do D2.append("-A DOCKER -d " ~ DOCKERMERGED.containers[container].ip ~ "/32 ! -i sobridge -o sobridge -p " ~ proto ~ " -m " ~ proto ~ " --dport " ~ containerPort ~ " -j ACCEPT") %}
 {%-     endfor %}
 {%-   endif %}
 {%- endfor %}
@@ -60,15 +52,11 @@
 :DOCKER - [0:0]
 -A PREROUTING -m addrtype --dst-type LOCAL -j DOCKER
 -A OUTPUT ! -d 127.0.0.0/8 -m addrtype --dst-type LOCAL -j DOCKER
-{%- for NETNAME in NODE_NETWORKS %}
--A POSTROUTING -s {{ DOCKERMERGED.networks[NETNAME].range }} ! -o {{ NETNAME }} -j MASQUERADE
-{%- endfor %}
+-A POSTROUTING -s {{DOCKERMERGED.range}} ! -o sobridge -j MASQUERADE
 {%- for rule in PR %}
 {{ rule }}
 {%- endfor %}
-{%- for NETNAME in NODE_NETWORKS %}
--A DOCKER -i {{ NETNAME }} -j RETURN
-{%- endfor %}
+-A DOCKER -i sobridge -j RETURN
 {%- for rule in D1 %}
 {{ rule }}
 {%- endfor %}
@@ -109,12 +97,10 @@ COMMIT
 {%- endif %}
 -A FORWARD -j DOCKER-USER
 -A FORWARD -j DOCKER-ISOLATION-STAGE-1
-{%- for NETNAME in NODE_NETWORKS %}
--A FORWARD -o {{ NETNAME }} -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
--A FORWARD -o {{ NETNAME }} -j DOCKER
--A FORWARD -i {{ NETNAME }} ! -o {{ NETNAME }} -j ACCEPT
--A FORWARD -i {{ NETNAME }} -o {{ NETNAME }} -j ACCEPT
-{%- endfor %}
+-A FORWARD -o sobridge -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
+-A FORWARD -o sobridge -j DOCKER
+-A FORWARD -i sobridge ! -o sobridge -j ACCEPT
+-A FORWARD -i sobridge -o sobridge -j ACCEPT
 -A FORWARD -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
 -A FORWARD -i lo -j ACCEPT
 -A FORWARD -m conntrack --ctstate INVALID -j DROP
@@ -126,18 +112,13 @@ COMMIT
 {%- for rule in D2 %}
 {{ rule }}
 {%- endfor %}
-{% for NETNAME in NODE_NETWORKS %}
--A DOCKER-ISOLATION-STAGE-1 -i {{ NETNAME }} ! -o {{ NETNAME }} -j DOCKER-ISOLATION-STAGE-2
-{%- endfor %}
+
+-A DOCKER-ISOLATION-STAGE-1 -i sobridge ! -o sobridge -j DOCKER-ISOLATION-STAGE-2
 -A DOCKER-ISOLATION-STAGE-1 -j RETURN
-{%- for NETNAME in NODE_NETWORKS %}
--A DOCKER-ISOLATION-STAGE-2 -o {{ NETNAME }} -j DROP
-{%- endfor %}
+-A DOCKER-ISOLATION-STAGE-2 -o sobridge -j DROP
 -A DOCKER-ISOLATION-STAGE-2 -j RETURN
-{%- for NETNAME in NODE_NETWORKS %}
--A DOCKER-USER ! -i {{ NETNAME }} -o {{ NETNAME }} -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
--A DOCKER-USER ! -i {{ NETNAME }} -o {{ NETNAME }} -j LOGGING
-{%- endfor %}
+-A DOCKER-USER ! -i sobridge -o sobridge -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
+-A DOCKER-USER ! -i sobridge -o sobridge -j LOGGING
 -A DOCKER-USER -j RETURN
 -A LOGGING -m limit --limit 2/min -j LOG --log-prefix "IPTables-dropped: "
 -A LOGGING -j DROP

salt/firewall/map.jinja

@@ -4,12 +4,8 @@
 
 {# add our ip to self #}
 {% do FIREWALL_DEFAULT.firewall.hostgroups.self.append(GLOBALS.node_ip) %}
-{# add dockernet ranges #}
-{% for NETNAME, NETWORK in DOCKERMERGED.networks.items() %}
-{%   if not NETWORK.get('manager_only') or GLOBALS.get('is_manager', False) %}
-{%     do FIREWALL_DEFAULT.firewall.hostgroups.dockernet.append(NETWORK.range) %}
-{%   endif %}
-{% endfor %}
+{# add dockernet range #}
+{% do FIREWALL_DEFAULT.firewall.hostgroups.dockernet.append(DOCKERMERGED.range) %}
 
 {% if GLOBALS.role == 'so-idh' %}
 {%   from 'idh/opencanary_config.map.jinja' import IDH_PORTGROUPS %}

salt/hydra/enabled.sls

@@ -26,8 +26,8 @@ so-hydra:
     - hostname: hydra
     - name: so-hydra
     - networks:
-      - soauth:
-        - ipv4_address: {{ DOCKERMERGED.containers['so-hydra'].ips['soauth'] }}
+      - sobridge:
+        - ipv4_address: {{ DOCKERMERGED.containers['so-hydra'].ip }}
     - binds:
       - /opt/so/conf/hydra/:/hydra-conf:ro
       - /opt/so/log/hydra/:/hydra-log:rw
@@ -73,7 +73,7 @@ delete_so-hydra_so-status.disabled:
 
 wait_for_hydra:
   http.wait_for_successful_query:
-    - name: 'http://{{ DOCKERMERGED.containers['so-hydra'].ips['soauth'] }}:4444/health/alive'
+    - name: 'http://{{ GLOBALS.manager }}:4444/health/alive'
     - ssl: True
     - verify_ssl: False
     - status:

salt/kratos/enabled.sls

@@ -19,8 +19,8 @@ so-kratos:
     - hostname: kratos
     - name: so-kratos
     - networks:
-      - soauth:
-        - ipv4_address: {{ DOCKERMERGED.containers['so-kratos'].ips['soauth'] }}
+      - sobridge:
+        - ipv4_address: {{ DOCKERMERGED.containers['so-kratos'].ip }}
     - binds:
       - /opt/so/conf/kratos/:/kratos-conf:ro
       - /opt/so/log/kratos/:/kratos-log:rw
@@ -71,7 +71,7 @@ delete_so-kratos_so-status.disabled:
 
 wait_for_kratos:
   http.wait_for_successful_query:
-    - name: 'http://{{ DOCKERMERGED.containers['so-kratos'].ips['soauth'] }}:4434/'
+    - name: 'http://{{ GLOBALS.manager }}:4434/'
     - ssl: True
     - verify_ssl: False
     - status:

salt/manager/tools/sbin/so-client

@@ -106,8 +106,7 @@ while [[ $# -gt 0 ]]; do
   esac
 done
 
-hydraContainer=${HYDRA_CONTAINER:-so-hydra}
-hydraUrl=${HYDRA_URL:-http://localhost:4445}
+hydraUrl=${HYDRA_URL:-http://127.0.0.1:4445}
 socRolesFile=${SOC_ROLES_FILE:-/opt/so/conf/soc/soc_clients_roles}
 soUID=${SOCORE_UID:-939}
 soGID=${SOCORE_GID:-939}
@@ -125,10 +124,6 @@ function fail() {
   exit 1
 }
 
-function hydraCurl() {
-  docker exec -i "$hydraContainer" curl "$@"
-}
-
 function require() {
   cmd=$1
   which "$1" 2>&1 > /dev/null
@@ -138,8 +133,8 @@ function require() {
 # Verify this environment is capable of running this script
 function verifyEnvironment() {
   require "jq"
-  require "docker"
-  response=$(hydraCurl -Ss -L ${hydraUrl}/health/alive)
+  require "curl"
+  response=$(curl -Ss -L ${hydraUrl}/health/alive)
   [[ "$response" != '{"status":"ok"}' ]] && fail "Unable to communicate with Hydra; specify URL via HYDRA_URL environment variable"
 }
 
@@ -169,7 +164,7 @@ function ensureRoleFileExists() {
 }
 
 function listClients() {
-  response=$(hydraCurl -Ss -L -f ${hydraUrl}/admin/clients)
+  response=$(curl -Ss -L -f ${hydraUrl}/admin/clients)
   [[ $? != 0 ]] && fail "Unable to communicate with Hydra"
 
   clientIds=$(echo "${response}" | jq -r ".[] | .client_id" | sort)
@@ -256,7 +251,7 @@ function createClient() {
 EOF
   )
   
-  response=$(hydraCurl -Ss -L --fail-with-body -X POST ${hydraUrl}/admin/clients -d "$body")
+  response=$(curl -Ss -L --fail-with-body -X POST ${hydraUrl}/admin/clients -d "$body")
   if [[ $? != 0 ]]; then
     error=$(echo $response | jq .error)
     fail "Failed to submit request to Hydra: $error"
@@ -288,7 +283,7 @@ function update() {
 EOF
   )
   
-  response=$(hydraCurl -Ss -L --fail-with-body -X PATCH ${hydraUrl}/admin/clients/$id -d "$body")
+  response=$(curl -Ss -L --fail-with-body -X PATCH ${hydraUrl}/admin/clients/$id -d "$body")
   if [[ $? != 0 ]]; then
     error=$(echo $response | jq .error)
     fail "Failed to submit request to Hydra: $error"
@@ -310,7 +305,7 @@ function generateSecret() {
 EOF
   )
   
-  response=$(hydraCurl -Ss -L --fail-with-body -X PATCH ${hydraUrl}/admin/clients/$id -d "$body")
+  response=$(curl -Ss -L --fail-with-body -X PATCH ${hydraUrl}/admin/clients/$id -d "$body")
   if [[ $? != 0 ]]; then
     error=$(echo $response | jq .error)
     fail "Failed to submit request to Hydra: $error"
@@ -322,7 +317,7 @@ function deleteClient() {
 
   [[ ${identityId} == "" ]] && fail "Client not found"
 
-  response=$(hydraCurl -Ss -XDELETE -L --fail-with-body "${hydraUrl}/admin/clients/$identityId")
+  response=$(curl -Ss -XDELETE -L --fail-with-body "${hydraUrl}/admin/clients/$identityId")
   if [[ $? != 0 ]]; then
     error=$(echo $response | jq .error)
     fail "Failed to submit request to Hydra: $error"

salt/manager/tools/sbin/so-user

@@ -129,8 +129,7 @@ while [[ $# -gt 0 ]]; do
   esac
 done
 
-kratosContainer=${KRATOS_CONTAINER:-so-kratos}
-kratosUrl=${KRATOS_URL:-http://localhost:4434/admin}
+kratosUrl=${KRATOS_URL:-http://127.0.0.1:4434/admin}
 databasePath=${KRATOS_DB_PATH:-/nsm/kratos/db/db.sqlite}
 databaseTimeout=${KRATOS_DB_TIMEOUT:-5000}
 bcryptRounds=${BCRYPT_ROUNDS:-12}
@@ -155,10 +154,6 @@ function fail() {
   exit 1
 }
 
-function kratosCurl() {
-  docker exec -i "$kratosContainer" curl "$@"
-}
-
 function require() {
   cmd=$1
   which "$1" 2>&1 > /dev/null
@@ -169,18 +164,18 @@ function require() {
 function verifyEnvironment() {
   require "htpasswd"
   require "jq"
-  require "docker"
+  require "curl"
   require "openssl"
   require "sqlite3"
   [[ ! -f $databasePath ]] && fail "Unable to find database file; specify path via KRATOS_DB_PATH environment variable"
-  response=$(kratosCurl -Ss -L ${kratosUrl}/)
+  response=$(curl -Ss -L ${kratosUrl}/)
   [[ "$response" != "404 page not found" ]] && fail "Unable to communicate with Kratos; specify URL via KRATOS_URL environment variable"
 }
 
 function findIdByEmail() {
   email=${1,,}
 
-  response=$(kratosCurl -Ss -L ${kratosUrl}/identities)
+  response=$(curl -Ss -L ${kratosUrl}/identities)
   identityId=$(echo "${response}" | jq -r ".[] | select(.verifiable_addresses[0].value == \"$email\") | .id")
   echo $identityId
 }
@@ -421,7 +416,7 @@ function syncAll() {
 }
 
 function listUsers() {
-  response=$(kratosCurl -Ss -L ${kratosUrl}/identities)
+  response=$(curl -Ss -L ${kratosUrl}/identities)
   [[ $? != 0 ]] && fail "Unable to communicate with Kratos"
 
   users=$(echo "${response}" | jq -r ".[] | .verifiable_addresses[0].value" | sort)
@@ -500,7 +495,7 @@ function createUser() {
 EOF
   )
   
-  response=$(kratosCurl -Ss -L ${kratosUrl}/identities -d "$addUserJson")
+  response=$(curl -Ss -L ${kratosUrl}/identities -d "$addUserJson")
   [[ $? != 0 ]] && fail "Unable to communicate with Kratos"
 
   identityId=$(echo "${response}" | jq -r ".id")
@@ -523,7 +518,7 @@ function updateStatus() {
   identityId=$(findIdByEmail "$email")
   [[ ${identityId} == "" ]] && fail "User not found"
 
-  response=$(kratosCurl -Ss -L "${kratosUrl}/identities/$identityId")
+  response=$(curl -Ss -L "${kratosUrl}/identities/$identityId")
   [[ $? != 0 ]] && fail "Unable to communicate with Kratos"
 
   schemaId=$(echo "$response" | jq -r .schema_id)
@@ -536,7 +531,7 @@ function updateStatus() {
     state="inactive"
   fi
   body="{ \"schema_id\": \"$schemaId\", \"state\": \"$state\", \"traits\": $traitBlock }"
-  response=$(kratosCurl -fSsL -XPUT -H "Content-Type: application/json" "${kratosUrl}/identities/$identityId" -d "$body")
+  response=$(curl -fSsL -XPUT -H "Content-Type: application/json" "${kratosUrl}/identities/$identityId" -d "$body")
   [[ $? != 0 ]] && fail "Unable to update user"
 }
 
@@ -555,7 +550,7 @@ function updateUserProfile() {
   identityId=$(findIdByEmail "$email")
   [[ ${identityId} == "" ]] && fail "User not found"
 
-  response=$(kratosCurl -Ss -L "${kratosUrl}/identities/$identityId")
+  response=$(curl -Ss -L "${kratosUrl}/identities/$identityId")
   [[ $? != 0 ]] && fail "Unable to communicate with Kratos"
 
   schemaId=$(echo "$response" | jq -r .schema_id)
@@ -564,7 +559,7 @@ function updateUserProfile() {
   traitBlock="{\"email\":\"$email\",\"firstName\":\"$firstName\",\"lastName\":\"$lastName\",\"note\":\"$note\"}"
 
   body="{ \"schema_id\": \"$schemaId\", \"state\": \"$state\", \"traits\": $traitBlock }"
-  response=$(kratosCurl -fSsL -XPUT -H "Content-Type: application/json" "${kratosUrl}/identities/$identityId" -d "$body")
+  response=$(curl -fSsL -XPUT -H "Content-Type: application/json" "${kratosUrl}/identities/$identityId" -d "$body")
   [[ $? != 0 ]] && fail "Unable to update user"
 }
 
@@ -574,7 +569,7 @@ function deleteUser() {
   identityId=$(findIdByEmail "$email")
   [[ ${identityId} == "" ]] && fail "User not found"
 
-  response=$(kratosCurl -Ss -XDELETE -L "${kratosUrl}/identities/$identityId")
+  response=$(curl -Ss -XDELETE -L "${kratosUrl}/identities/$identityId")
   [[ $? != 0 ]] && fail "Unable to communicate with Kratos"
 
   rolesTmpFile="${socRolesFile}.tmp"

salt/manager/tools/sbin/soup

@@ -28,7 +28,6 @@ INSTALLEDSALTVERSION=$(salt --versions-report | grep Salt: | awk '{print $2}')
 # percentage like "25%"). Empty means so-soup-grid-highstate uses the salt:auto_apply:batch
 # pillar default.
 BATCHSIZE=
-DEFAULT_DOCKER_RANGE='172.17.1.0/24'
 SOUP_LOG=/root/soup.log
 SOUP_DEBUG_LOG=/root/soup-debug.log
 WHATWOULDYOUSAYYAHDOHERE=soup
@@ -526,7 +525,6 @@ preupgrade_changes() {
     [[ "$INSTALLEDVERSION" == "3.0.0" ]] && up_to_3.1.0
     [[ "$INSTALLEDVERSION" == "3.1.0" ]] && up_to_3.2.0
     [[ "$INSTALLEDVERSION" == "3.2.0" ]] && up_to_3.3.0
-    [[ "$INSTALLEDVERSION" == "3.3.0" ]] && up_to_3.4.0
     true
 }
 
@@ -545,7 +543,6 @@ postupgrade_changes() {
     [[ "$POSTVERSION" == "3.0.0" ]] && post_to_3.1.0
     [[ "$POSTVERSION" == "3.1.0" ]] && post_to_3.2.0
     [[ "$POSTVERSION" == "3.2.0" ]] && post_to_3.3.0
-    [[ "$POSTVERSION" == "3.3.0" ]] && post_to_3.4.0
     # All applicable post-upgrade steps completed; clear the resume marker.
     rm -f "$POSTVERSION_FILE"
     true
@@ -1096,82 +1093,6 @@ post_to_3.3.0() {
 }
 ### 3.3.0 End ###
 
-### 3.4.0 Scripts ###
-up_to_3.4.0() {
-  set_soauth_range
-
-  echo "Removing so-kratos, so-hydra and so-soc so they are recreated on the soauth network."
-  docker rm -f so-kratos so-hydra so-soc >> $SOUP_LOG 2>&1
-
-  INSTALLEDVERSION=3.4.0
-}
-
-set_soauth_range() {
-  local pillar_file=/opt/so/saltstack/local/pillar/docker/soc_docker.sls
-  local current_range suggested authnet authgw input
-
-  [[ -f "$pillar_file" ]] || return 0
-
-  current_range=$(so-yaml.py get -r "$pillar_file" docker.range 2>/dev/null) || return 0
-
-  # A default range gets the 172.17.2.0/24 from docker/defaults.yaml, same as a fresh
-  # install, so there is nothing to ask about.
-  [[ -n "$current_range" && "$current_range" != "$DEFAULT_DOCKER_RANGE" ]] || return 0
-
-  if so-yaml.py get -r "$pillar_file" docker.networks.soauth.range >/dev/null 2>&1; then
-    return 0
-  fi
-
-  suggested=$(echo "${current_range%%/*}" | awk -F'.' '{ printf "%s.%s.%s.%s", $1, $2, ($3 + 1) % 256, $4 }')
-
-  if [[ -z $UNATTENDED ]]; then
-    echo ""
-    echo "This grid uses a custom Docker range ($current_range). The authentication"
-    echo "services are moving to their own isolated network, which needs a second /24"
-    echo "that does not overlap it."
-    echo ""
-    while :; do
-      read -rp "Enter the network without the /24 suffix, or press Enter for ${suggested}: " input
-      [[ -z "$input" ]] && input="$suggested"
-      if valid_soauth_range "$input" "$current_range"; then
-        authnet="$input"
-        break
-      fi
-      echo "That range must be a valid IPv4 network, must not be within 172.17.0.0/24, and must not overlap ${current_range}."
-    done
-  else
-    if ! valid_soauth_range "$suggested" "$current_range"; then
-      FINAL_MESSAGE_QUEUE+=("WARNING: Unable to pick a range for the authentication network alongside $current_range. Set it manually before the next highstate:")
-      FINAL_MESSAGE_QUEUE+=("    - so-yaml.py add $pillar_file docker.networks.soauth.range <network>/24")
-      FINAL_MESSAGE_QUEUE+=("    - so-yaml.py add $pillar_file docker.networks.soauth.gateway <gateway>")
-      return 0
-    fi
-    authnet="$suggested"
-    FINAL_MESSAGE_QUEUE+=("NOTE: The authentication services moved to an isolated Docker network and were assigned ${authnet}/24.")
-    FINAL_MESSAGE_QUEUE+=("    - If that conflicts with your environment, update docker.networks.soauth in $pillar_file and run so-checkin.")
-  fi
-
-  authgw=$(echo "$authnet" | awk -F'.' '{print $1,$2,$3,1}' OFS='.')
-
-  echo "Assigning the authentication network the range ${authnet}/24."
-  so-yaml.py add "$pillar_file" docker.networks.soauth.range "${authnet}/24" >> $SOUP_LOG 2>&1
-  so-yaml.py add "$pillar_file" docker.networks.soauth.gateway "$authgw" >> $SOUP_LOG 2>&1
-}
-
-valid_soauth_range() {
-  local candidate=$1 docker_range=$2
-
-  valid_ip4 "$candidate" || return 1
-  [[ $candidate =~ ^172\.17\.0\. ]] && return 1
-  [[ "${candidate}/24" == "$docker_range" ]] && return 1
-  return 0
-}
-
-post_to_3.4.0() {
-  set_postversion 3.4.0
-}
-### 3.4.0 End ###
-
 
 repo_sync() {
   echo "Sync the local repo."

salt/nginx/etc/nginx.conf

@@ -183,7 +183,7 @@ http {
 		ssl_prefer_server_ciphers on;
 		ssl_protocols TLSv1.2 TLSv1.3;
 
-		location ~* (^/login/.*|^/js/.*|^/css/.*|^/images/.*|^/pages/.*|^/docs/.*) {
+		location ~* (^/login|^/login/.*|^/js/.*|^/css/.*|^/images/.*|^/pages/.*|^/docs/.*) {
 			proxy_pass            http://{{ GLOBALS.manager }}:9822;
 			proxy_read_timeout    90;
 			proxy_connect_timeout 90;

salt/soc/defaults.map.jinja

@@ -14,8 +14,6 @@
 {%   do SOCDEFAULTS.soc.config.server.modules[module].update({'hostUrl': application_url}) %}
 {% endfor %}
 
-{% do SOCDEFAULTS.soc.config.server.modules.kratos.update({'publicHostUrl': 'http://' ~ DOCKERMERGED.containers['so-kratos'].ips['soauth'] ~ ':4433/'}) %}
-
 {# add all grid heavy nodes to soc.server.modules.elastic.remoteHostUrls #}
 {% for node_type, minions in salt['pillar.get']('elasticsearch:nodes', {}).items() %}
 {%   if node_type in ['heavynode'] %}

salt/soc/defaults.yaml

@@ -1380,7 +1380,6 @@ soc:
           retryFailureMaxAttempts: 5
         kratos:
           hostUrl:
-          publicHostUrl:
         hydra:
           hostUrl:
         elastalertengine:
@@ -1557,6 +1556,69 @@ soc:
           reconcilePersona: ""
           toolUseTurnAttempts: 12
           toolUseTurnDelayMs: 175
+          tools:
+            filterEventFields:
+              - "@timestamp"
+              - "client.name"
+              - "destination.ip"
+              - "destination.port"
+              - "destination.geo.country_name"
+              - "dns.query.name"
+              - "dns.query_name"
+              - "event.action"
+              - "event.category"
+              - "event.module"
+              - "event.dataset"
+              - "event.outcome"
+              - "event.severity"
+              - "event.severity_label"
+              - "event.type"
+              - "event_data.agent.name"
+              - "event_data.host.os.name"
+              - "file.mime_type"
+              - "file.name"
+              - "hash.md5"
+              - "hash.sha1"
+              - "host.mac"
+              - "host.name"
+              - "host.os.name"
+              - "http.method"
+              - "http.useragent"
+              - "http.virtual_host"
+              - "log.id.uid"
+              - "network.community_id"
+              - "network.protocol"
+              - "network.transport"
+              - "notice.message"
+              - "observer.name"
+              - "process.name"
+              - "process.executable"
+              - "process.entity_id"
+              - "process.command_line"
+              - "process.Ext.ancestry"
+              - "process.parent.entity_id"
+              - "process.parent.command_line"
+              - "rule.category"
+              - "rule.name"
+              - "rule.uuid"
+              - "software.name"
+              - "software.type"
+              - "software.version.unparsed"
+              - "source.ip"
+              - "source.port"
+              - "source.geo.country_name"
+              - "ssh.cypher_algorithm"
+              - "ssh.client"
+              - "ssh.server"
+              - "ssl.cipher"
+              - "ssl.server_name"
+              - "ssl.version"
+              - "system.auth.sudo.command"
+              - "user.name"
+              - "user.domain"
+              - "user.effective.name"
+              - "weird.name"
+              - "tags"
         onionconfig:
           saltstackDir: /opt/so/saltstack
           bypassEnabled: false

salt/soc/enabled.sls

@@ -23,9 +23,7 @@ so-soc:
     - name: so-soc
     - networks:
       - sobridge:
-        - ipv4_address: {{ DOCKERMERGED.containers['so-soc'].ips['sobridge'] }}
-      - soauth:
-        - ipv4_address: {{ DOCKERMERGED.containers['so-soc'].ips['soauth'] }}
+        - ipv4_address: {{ DOCKERMERGED.containers['so-soc'].ip }}
     - binds:
       - /nsm/rules:/nsm/rules:rw
       - /opt/so/conf/strelka:/opt/sensoroni/yara:rw

salt/soc/soc_soc.yaml

@@ -916,6 +916,11 @@ soc:
             description: The number of times to retry extracting memories from a session if errors occur.
             global: True
             advanced: True
+          tools:
+            filterEventFields:
+              description: A whitelist of fields to return when OnionAI uses the query_events tool. All other fields are removed. One field per line.
+              global: True
+              multiline: True
       client:
         assistant:
           enabled:

salt/vars/globals.map.jinja

@@ -54,8 +54,8 @@
 {%
   do GLOBALS.update({
     'application_urls': {
-      'hydra': 'http://' ~ DOCKERMERGED.containers['so-hydra'].ips['soauth'] ~ ':4445/',
-      'kratos': 'http://' ~ DOCKERMERGED.containers['so-kratos'].ips['soauth'] ~ ':4434/',
+      'hydra': 'http://' ~ GLOBALS.manager ~ ':4445/',
+      'kratos': 'http://' ~ GLOBALS.manager ~ ':4434/',
       'elastic': 'https://' ~ GLOBALS.manager ~ ':9200/',
       'influxdb': 'https://' ~ GLOBALS.manager ~ ':8086/'
     }

salt/zeek/defaults.yaml

@@ -18,6 +18,7 @@ zeek:
       StatsLogEnable: 0
       StatsLogExpireInterval: 0
       StatusCmdShowAll: 0
+      StopWait: 1
       CrashExpireInterval: 0
       SitePolicyScripts: local.zeek
       LogDir: /nsm/zeek/logs

salt/zeek/disabled.sls

@@ -9,9 +9,19 @@
 include:
   - zeek.sostatus
   
+# Stop first so the entrypoint's SIGTERM trap can archive the final logs; docker_container.absent
+# with force is a 'docker rm -f', which never delivers SIGTERM. force stays so the state still
+# converges if the stop overruns.
+so-zeek_stopped:
+  docker_container.stopped:
+    - name: so-zeek
+    - error_on_absent: False
+
 so-zeek:
   docker_container.absent:
     - force: True
+    - require:
+      - docker_container: so-zeek_stopped
 
 so-zeek_so-status.disabled:
   file.comment:

salt/zeek/enabled.sls

@@ -19,6 +19,10 @@ so-zeek:
     - restart_policy: unless-stopped
     - start: True
     - privileged: True
+    # Docker's default 10s grace is not enough for the entrypoint's SIGTERM trap to run
+    # 'zeekctl stop' and let StopWait archive the final logs. Overrunning it means SIGKILL,
+    # which strands those logs in spool/tmp and marks every node crashed on the next start.
+    - stop_timeout: 180
     {% if DOCKERMERGED.containers['so-zeek'].ulimits %}
     - ulimits:
     {%   for ULIMIT in DOCKERMERGED.containers['so-zeek'].ulimits %}

salt/zeek/soc_zeek.yaml

@@ -99,6 +99,18 @@ zeek:
         regexFailureMessage: You must enter a whole number of days, or 0 to keep crash directories forever.
         helpLink: zeek
         advanced: True
+      StopWait:
+        description: >-
+          Set to 1 to make "zeekctl stop" wait for the final logs to be archived instead of
+          letting that finish in the background. Security Onion stops Zeek by stopping its
+          container, so anything still running in the background is killed when the container
+          exits - without this, the last logs of each run are stranded unarchived in
+          /nsm/zeek/spool/tmp and never reach Elasticsearch. It is read only for that reason.
+        regex: ^[01]$
+        regexFailureMessage: You must enter 0 or 1.
+        helpLink: zeek
+        advanced: True
+        readonly: True
       MinDiskSpace:
         description: >-
           Percentage of free disk space below which ZeekControl reports a warning, or 0 to disable the check

setup/so-functions

@@ -276,20 +276,9 @@ collect_dockernet() {
 			whiptail_invalid_input
 			whiptail_dockernet_sosnet "$DOCKERNET"
 		done
-
-		whiptail_authnet_sosnet "$(adjacent_net "$DOCKERNET")"
-
-		while ! valid_ip4 "$AUTHNET" || [[ $AUTHNET =~ "172.17.0." ]] || [[ "$AUTHNET" == "$DOCKERNET" ]]; do
-			whiptail_invalid_input
-			whiptail_authnet_sosnet "$AUTHNET"
-		done
 	fi
 }
 
-adjacent_net() {
-	echo "$1" | awk -F'.' '{ printf "%s.%s.%s.%s", $1, $2, ($3 + 1) % 256, $4 }'
-}
-
 collect_gateway() {
 	whiptail_management_interface_gateway
 
@@ -1410,15 +1399,6 @@ docker_pillar() {
 			"docker:"\
 			"  range: '$DOCKERNET/24'"\
 			"  gateway: '$DOCKERGATEWAY'" > $docker_pillar_file
-
-		if [ ! -z "$AUTHNET" ]; then
-			AUTHGATEWAY=$(echo $AUTHNET | awk -F'.' '{print $1,$2,$3,1}' OFS='.')
-			printf '%s\n'\
-				"  networks:"\
-				"    soauth:"\
-				"      range: '$AUTHNET/24'"\
-				"      gateway: '$AUTHGATEWAY'" >> $docker_pillar_file
-		fi
 	fi
 }
 

setup/so-whiptail

@@ -365,18 +365,6 @@ whiptail_dockernet_sosnet() {
 
 }
 
-whiptail_authnet_sosnet() {
-
-	[ -n "$TESTING" ] && return
-
-	AUTHNET=$(whiptail --title "$whiptail_title" --inputbox \
-	"\nEnter a second /24 size network range WITHOUT the /24 suffix. The authentication services are isolated on their own network so that the identity provider is not reachable from other containers. It must not overlap the range you just entered, and any range within 172.17.0.0/24 cannot be used." 13 65 "$1" 3>&1 1>&2 2>&3)
-
-	local exitstatus=$?
-	whiptail_check_exitstatus $exitstatus
-
-}
-
 whiptail_end_settings() {
 	[ -n "$TESTING" ] && return
 
@@ -439,7 +427,6 @@ whiptail_end_settings() {
 	[[ -n $WEBUSER ]] && __append_end_msg "Web User: $WEBUSER"
 
 	[[ -n $DOCKERNET ]] && __append_end_msg "Docker network: $DOCKERNET/24"
-	[[ -n $AUTHNET ]] && __append_end_msg "Authentication network: $AUTHNET/24"
 	if [[ ${#ntp_servers[@]} -gt 0 ]]; then
 		__append_end_msg "NTP Servers:"
 		for server in "${ntp_servers[@]}"; do