Gracefully skip SSL cert states when the CA is unavailable

The *.ssl x509.certificate_managed states hard-failed the state run when
the CA/manager minion was unreachable (ca_server did not respond), and the
paired private_key_managed states then failed on the prereq requisite. The
- timeout: 30 on these states was a no-op (Salt has no per-state timeout
requisite; only retry accepts until/attempts/interval/splay).

Add an onlyif CA-reachability gate to every certificate_managed state so the
state is skipped as a clean success when the CA does not answer a peer
test.ping within 3s and the cert already exists. When the cert is missing
(initial provisioning) the state still runs so the existing retry waits for
the CA. The reachability command is centralized as CA.reachable_cmd in
ca/map.jinja, and the master peer config now permits test.ping. The no-op
- timeout: 30 is removed from all cert states.

files/salt/master/master

@@ -64,5 +64,6 @@ pillar_roots:
 peer:
   .*:
     - x509.sign_remote_certificate
+    - test.ping
 
 

salt/ca/map.jinja

@@ -1,3 +1,7 @@
 {% set CA = {
     'server': pillar.ca.server
 }%}
+{# reachable_cmd: shell test used by the *.ssl cert states' onlyif to skip cert
+   management gracefully when the CA minion is unreachable. Returns exit 0 when
+   the CA answers a peer test.ping within 3s, non-zero otherwise. #}
+{% do CA.update({'reachable_cmd': "salt-call --out=json publish.publish '" ~ CA.server ~ "' test.ping timeout=3 2>/dev/null | grep -q true"}) %}

salt/common/tools/sbin/so-log-check

@@ -132,7 +132,6 @@ if [[ $EXCLUDE_STARTUP_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|HTTP 404: Not Found"          # Salt loops until Kratos returns 200, during startup Kratos may not be ready
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Cancelling deferred write event maybeFenceReplicas because the event queue is now closed" # Kafka controller log during shutdown/restart
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Redis may have been restarted" # Redis likely restarted by salt
-    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|file already closed"          # Go logging race condition during container restart
 fi
 
 if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then

salt/elasticfleet/ssl.sls

@@ -38,7 +38,8 @@ etc_elasticfleet_crt:
     - days_remaining: 7
     - days_valid: 820
     - backup: True
-    - timeout: 30
+    - onlyif:
+      - test ! -f /etc/pki/elasticfleet-server.crt || {{ CA.reachable_cmd }}
     - retry:
         attempts: 5
         interval: 30
@@ -94,7 +95,8 @@ etc_elasticfleet_agent_crt:
     - days_remaining: 7
     - days_valid: 820
     - backup: True
-    - timeout: 30
+    - onlyif:
+      - test ! -f /etc/pki/elasticfleet-agent.crt || {{ CA.reachable_cmd }}
     - retry:
         attempts: 5
         interval: 30
@@ -155,7 +157,8 @@ elasticfleet_kafka_crt:
     - days_remaining: 7
     - days_valid: 820
     - backup: True
-    - timeout: 30
+    - onlyif:
+      - test ! -f /etc/pki/elasticfleet-kafka.crt || {{ CA.reachable_cmd }}
     - retry:
         attempts: 5
         interval: 30

salt/elasticsearch/ssl.sls

@@ -34,7 +34,8 @@ elasticsearch_crt:
     - days_remaining: 7
     - days_valid: 820
     - backup: True
-    - timeout: 30
+    - onlyif:
+      - test ! -f /etc/pki/elasticsearch.crt || {{ CA.reachable_cmd }}
     - retry:
         attempts: 5
         interval: 30

salt/influxdb/ssl.sls

@@ -34,7 +34,8 @@ influxdb_crt:
     - days_remaining: 7
     - days_valid: 820
     - backup: True
-    - timeout: 30
+    - onlyif:
+      - test ! -f /etc/pki/influxdb.crt || {{ CA.reachable_cmd }}
     - retry:
         attempts: 5
         interval: 30

salt/kafka/ssl.sls

@@ -38,7 +38,8 @@ kafka_client_crt:
     - days_remaining: 7
     - days_valid: 820
     - backup: True
-    - timeout: 30
+    - onlyif:
+      - test ! -f /etc/pki/kafka-client.crt || {{ CA.reachable_cmd }}
     - retry:
         attempts: 5
         interval: 30
@@ -86,7 +87,8 @@ kafka_crt:
     - days_remaining: 7
     - days_valid: 820
     - backup: True
-    - timeout: 30
+    - onlyif:
+      - test ! -f /etc/pki/kafka.crt || {{ CA.reachable_cmd }}
     - retry:
         attempts: 5
         interval: 30
@@ -148,7 +150,8 @@ kafka_logstash_crt:
     - days_remaining: 7
     - days_valid: 820
     - backup: True
-    - timeout: 30
+    - onlyif:
+      - test ! -f /etc/pki/kafka-logstash.crt || {{ CA.reachable_cmd }}
     - retry:
         attempts: 5
         interval: 30

salt/logstash/ssl.sls

@@ -38,7 +38,8 @@ etc_elasticfleet_logstash_crt:
     - days_remaining: 7
     - days_valid: 820
     - backup: True
-    - timeout: 30
+    - onlyif:
+      - test ! -f /etc/pki/elasticfleet-logstash.crt || {{ CA.reachable_cmd }}
     - retry:
         attempts: 5
         interval: 30
@@ -99,7 +100,8 @@ etc_elasticfleetlumberjack_crt:
     - days_remaining: 7
     - days_valid: 820
     - backup: True
-    - timeout: 30
+    - onlyif:
+      - test ! -f /etc/pki/elasticfleet-lumberjack.crt || {{ CA.reachable_cmd }}
     - retry:
         attempts: 5
         interval: 30
@@ -168,7 +170,8 @@ etc_filebeat_crt:
     - days_remaining: 7
     - days_valid: 820
     - backup: True
-    - timeout: 30
+    - onlyif:
+      - test ! -f /etc/pki/filebeat.crt || {{ CA.reachable_cmd }}
     - retry:
         attempts: 5
         interval: 30
@@ -249,7 +252,8 @@ conf_filebeat_crt:
     - days_remaining: 7
     - days_valid: 820
     - backup: True
-    - timeout: 30
+    - onlyif:
+      - test ! -f /opt/so/conf/filebeat/etc/pki/filebeat.crt || {{ CA.reachable_cmd }}
     - retry:
         attempts: 5
         interval: 30

salt/manager/tools/sbin/soup

@@ -437,8 +437,6 @@ get_soup_script_hashes() {
   GITIMGCMN=$(md5sum $UPDATE_DIR/salt/common/tools/sbin/so-image-common | awk '{print $1}')
   CURRENTSOFIREWALL=$(md5sum /usr/sbin/so-firewall | awk '{print $1}')
   GITSOFIREWALL=$(md5sum $UPDATE_DIR/salt/manager/tools/sbin/so-firewall | awk '{print $1}')
-  CURRENTSOYAML=$(md5sum /usr/sbin/so-yaml.py | awk '{print $1}')
-  GITSOYAML=$(md5sum $UPDATE_DIR/salt/manager/tools/sbin/so-yaml.py | awk '{print $1}')
 }
 
 highstate() {
@@ -1226,7 +1224,7 @@ upgrade_salt() {
 
 verify_latest_update_script() {
   get_soup_script_hashes
-  if [[ "$CURRENTSOUP" == "$GITSOUP" && "$CURRENTCMN" == "$GITCMN" && "$CURRENTIMGCMN" == "$GITIMGCMN" && "$CURRENTSOFIREWALL" == "$GITSOFIREWALL" && "$CURRENTSOYAML" == "$GITSOYAML" ]]; then
+  if [[ "$CURRENTSOUP" == "$GITSOUP" && "$CURRENTCMN" == "$GITCMN" && "$CURRENTIMGCMN" == "$GITIMGCMN" && "$CURRENTSOFIREWALL" == "$GITSOFIREWALL" ]]; then
     echo "This version of the soup script is up to date. Proceeding."
   else
     echo "You are not running the latest soup version. Updating soup and its components. This might take multiple runs to complete."
@@ -1235,7 +1233,7 @@ verify_latest_update_script() {
 
     # Verify that soup scripts updated as expected
     get_soup_script_hashes
-    if [[ "$CURRENTSOUP" == "$GITSOUP" && "$CURRENTCMN" == "$GITCMN" && "$CURRENTIMGCMN" == "$GITIMGCMN" && "$CURRENTSOFIREWALL" == "$GITSOFIREWALL" && "$CURRENTSOYAML" == "$GITSOYAML" ]]; then
+    if [[ "$CURRENTSOUP" == "$GITSOUP" && "$CURRENTCMN" == "$GITCMN" && "$CURRENTIMGCMN" == "$GITIMGCMN" && "$CURRENTSOFIREWALL" == "$GITSOFIREWALL" ]]; then
       echo "Succesfully updated soup scripts."
     else
       echo "There was a problem updating soup scripts. Trying to rerun script update."

salt/nginx/ssl.sls

@@ -71,7 +71,8 @@ managerssl_crt:
     - days_remaining: 7
     - days_valid: 820
     - backup: True
-    - timeout: 30
+    - onlyif:
+      - test ! -f /etc/pki/managerssl.crt || {{ CA.reachable_cmd }}
     - retry:
         attempts: 5
         interval: 30

salt/postgres/ssl.sls

@@ -33,7 +33,8 @@ postgres_crt:
     - days_remaining: 7
     - days_valid: 820
     - backup: True
-    - timeout: 30
+    - onlyif:
+      - test ! -f /etc/pki/postgres.crt || {{ CA.reachable_cmd }}
     - retry:
         attempts: 5
         interval: 30

salt/redis/ssl.sls

@@ -33,7 +33,8 @@ redis_crt:
     - days_remaining: 7
     - days_valid: 820
     - backup: True
-    - timeout: 30
+    - onlyif:
+      - test ! -f /etc/pki/redis.crt || {{ CA.reachable_cmd }}
     - retry:
         attempts: 5
         interval: 30

salt/registry/ssl.sls

@@ -55,7 +55,8 @@ registry_crt:
     - backup: True
     - require:
       - file: registry_crt_cleanup
-    - timeout: 30
+    - onlyif:
+      - test ! -f /etc/pki/registry.crt || {{ CA.reachable_cmd }}
     - retry:
         attempts: 15
         interval: 10

salt/soc/defaults.yaml

@@ -1531,8 +1531,7 @@ soc:
           agentic: false
           agentMapping:
             Orchestrator: sonnet
-            Investigator: sonnet
-            Detection Engineer: sonnet
+            Hunter: sonnet
         onionconfig:
           saltstackDir: /opt/so/saltstack
           bypassEnabled: false

salt/soc/soc_soc.yaml

@@ -783,11 +783,8 @@ soc:
             Orchestrator:
               description: The initial agent in most agentic conversations. This agent will delegate requests to specialized agents.
               global: True
-            Investigator:
-              description: This agent investigates alerts, explains events and records, and hunts through event data. It can also acknowledge alerts and escalate to cases.
-              global: True
-            Detection Engineer:
-              description: This agent manages detections and their overrides, including tuning noisy rules and authoring rule content.
+            Hunter:
+              description: This agent is specialized in querying events.
               global: True
       client:
         assistant:

salt/suricata/soc_suricata.yaml

@@ -153,12 +153,12 @@ suricata:
       cpu-affinity:
         management-cpu-set:
           cpu:
-            description: Bind management threads to a core or range of cores. This can be a single core, list of cores, or list of range of cores. set-cpu-affinity must be set to true for this to be used.
+            description: Bind management threads to a core or range of cores. This can be a sigle core, list of cores, or list of range of cores. set-cpu-affinity must be set to true for this to be used.
             forcedType: "[]string"
             helpLink: suricata
         worker-cpu-set:
           cpu:
-            description: Bind worker threads to a core or range of cores. This can be a single core, list of cores, or list of range of cores. set-cpu-affinity must be set to true for this to be used.
+            description: Bind worker threads to a core or range of cores. This can be a sigle core, list of cores, or list of range of cores. set-cpu-affinity must be set to true for this to be used.
             forcedType: "[]string"
             helpLink: suricata
     vars:

salt/telegraf/ssl.sls

@@ -34,7 +34,8 @@ telegraf_crt:
     - days_remaining: 7
     - days_valid: 820
     - backup: True
-    - timeout: 30
+    - onlyif:
+      - test ! -f /etc/pki/telegraf.crt || {{ CA.reachable_cmd }}
     - retry:
         attempts: 5
         interval: 30