Gracefully skip SSL cert states when the CA is unavailable

The *.ssl x509.certificate_managed states hard-failed the state run when
the CA/manager minion was unreachable (ca_server did not respond), and the
paired private_key_managed states then failed on the prereq requisite. The
- timeout: 30 on these states was a no-op (Salt has no per-state timeout
requisite; only retry accepts until/attempts/interval/splay).

Add an onlyif CA-reachability gate to every certificate_managed state so the
state is skipped as a clean success when the CA does not answer a peer
test.ping within 3s and the cert already exists. When the cert is missing
(initial provisioning) the state still runs so the existing retry waits for
the CA. The reachability command is centralized as CA.reachable_cmd in
ca/map.jinja, and the master peer config now permits test.ping. The no-op
- timeout: 30 is removed from all cert states.

files/salt/master/master

@@ -64,5 +64,6 @@ pillar_roots:
 peer:
   .*:
     - x509.sign_remote_certificate
+    - test.ping
 
 

salt/ca/map.jinja

@@ -1,3 +1,7 @@
 {% set CA = {
     'server': pillar.ca.server
 }%}
+{# reachable_cmd: shell test used by the *.ssl cert states' onlyif to skip cert
+   management gracefully when the CA minion is unreachable. Returns exit 0 when
+   the CA answers a peer test.ping within 3s, non-zero otherwise. #}
+{% do CA.update({'reachable_cmd': "salt-call --out=json publish.publish '" ~ CA.server ~ "' test.ping timeout=3 2>/dev/null | grep -q true"}) %}

salt/elasticfleet/ssl.sls

@@ -38,7 +38,8 @@ etc_elasticfleet_crt:
     - days_remaining: 7
     - days_valid: 820
     - backup: True
-    - timeout: 30
+    - onlyif:
+      - test ! -f /etc/pki/elasticfleet-server.crt || {{ CA.reachable_cmd }}
     - retry:
         attempts: 5
         interval: 30
@@ -94,7 +95,8 @@ etc_elasticfleet_agent_crt:
     - days_remaining: 7
     - days_valid: 820
     - backup: True
-    - timeout: 30
+    - onlyif:
+      - test ! -f /etc/pki/elasticfleet-agent.crt || {{ CA.reachable_cmd }}
     - retry:
         attempts: 5
         interval: 30
@@ -155,7 +157,8 @@ elasticfleet_kafka_crt:
     - days_remaining: 7
     - days_valid: 820
     - backup: True
-    - timeout: 30
+    - onlyif:
+      - test ! -f /etc/pki/elasticfleet-kafka.crt || {{ CA.reachable_cmd }}
     - retry:
         attempts: 5
         interval: 30

salt/elasticsearch/ssl.sls

@@ -34,7 +34,8 @@ elasticsearch_crt:
     - days_remaining: 7
     - days_valid: 820
     - backup: True
-    - timeout: 30
+    - onlyif:
+      - test ! -f /etc/pki/elasticsearch.crt || {{ CA.reachable_cmd }}
     - retry:
         attempts: 5
         interval: 30

salt/influxdb/ssl.sls

@@ -34,7 +34,8 @@ influxdb_crt:
     - days_remaining: 7
     - days_valid: 820
     - backup: True
-    - timeout: 30
+    - onlyif:
+      - test ! -f /etc/pki/influxdb.crt || {{ CA.reachable_cmd }}
     - retry:
         attempts: 5
         interval: 30

salt/kafka/ssl.sls

@@ -38,7 +38,8 @@ kafka_client_crt:
     - days_remaining: 7
     - days_valid: 820
     - backup: True
-    - timeout: 30
+    - onlyif:
+      - test ! -f /etc/pki/kafka-client.crt || {{ CA.reachable_cmd }}
     - retry:
         attempts: 5
         interval: 30
@@ -86,7 +87,8 @@ kafka_crt:
     - days_remaining: 7
     - days_valid: 820
     - backup: True
-    - timeout: 30
+    - onlyif:
+      - test ! -f /etc/pki/kafka.crt || {{ CA.reachable_cmd }}
     - retry:
         attempts: 5
         interval: 30
@@ -148,7 +150,8 @@ kafka_logstash_crt:
     - days_remaining: 7
     - days_valid: 820
     - backup: True
-    - timeout: 30
+    - onlyif:
+      - test ! -f /etc/pki/kafka-logstash.crt || {{ CA.reachable_cmd }}
     - retry:
         attempts: 5
         interval: 30

salt/logstash/ssl.sls

@@ -38,7 +38,8 @@ etc_elasticfleet_logstash_crt:
     - days_remaining: 7
     - days_valid: 820
     - backup: True
-    - timeout: 30
+    - onlyif:
+      - test ! -f /etc/pki/elasticfleet-logstash.crt || {{ CA.reachable_cmd }}
     - retry:
         attempts: 5
         interval: 30
@@ -99,7 +100,8 @@ etc_elasticfleetlumberjack_crt:
     - days_remaining: 7
     - days_valid: 820
     - backup: True
-    - timeout: 30
+    - onlyif:
+      - test ! -f /etc/pki/elasticfleet-lumberjack.crt || {{ CA.reachable_cmd }}
     - retry:
         attempts: 5
         interval: 30
@@ -168,7 +170,8 @@ etc_filebeat_crt:
     - days_remaining: 7
     - days_valid: 820
     - backup: True
-    - timeout: 30
+    - onlyif:
+      - test ! -f /etc/pki/filebeat.crt || {{ CA.reachable_cmd }}
     - retry:
         attempts: 5
         interval: 30
@@ -249,7 +252,8 @@ conf_filebeat_crt:
     - days_remaining: 7
     - days_valid: 820
     - backup: True
-    - timeout: 30
+    - onlyif:
+      - test ! -f /opt/so/conf/filebeat/etc/pki/filebeat.crt || {{ CA.reachable_cmd }}
     - retry:
         attempts: 5
         interval: 30

salt/nginx/ssl.sls

@@ -71,7 +71,8 @@ managerssl_crt:
     - days_remaining: 7
     - days_valid: 820
     - backup: True
-    - timeout: 30
+    - onlyif:
+      - test ! -f /etc/pki/managerssl.crt || {{ CA.reachable_cmd }}
     - retry:
         attempts: 5
         interval: 30

salt/postgres/ssl.sls

@@ -33,7 +33,8 @@ postgres_crt:
     - days_remaining: 7
     - days_valid: 820
     - backup: True
-    - timeout: 30
+    - onlyif:
+      - test ! -f /etc/pki/postgres.crt || {{ CA.reachable_cmd }}
     - retry:
         attempts: 5
         interval: 30

salt/redis/ssl.sls

@@ -33,7 +33,8 @@ redis_crt:
     - days_remaining: 7
     - days_valid: 820
     - backup: True
-    - timeout: 30
+    - onlyif:
+      - test ! -f /etc/pki/redis.crt || {{ CA.reachable_cmd }}
     - retry:
         attempts: 5
         interval: 30

salt/registry/ssl.sls

@@ -55,7 +55,8 @@ registry_crt:
     - backup: True
     - require:
       - file: registry_crt_cleanup
-    - timeout: 30
+    - onlyif:
+      - test ! -f /etc/pki/registry.crt || {{ CA.reachable_cmd }}
     - retry:
         attempts: 15
         interval: 10

salt/telegraf/ssl.sls

@@ -34,7 +34,8 @@ telegraf_crt:
     - days_remaining: 7
     - days_valid: 820
     - backup: True
-    - timeout: 30
+    - onlyif:
+      - test ! -f /etc/pki/telegraf.crt || {{ CA.reachable_cmd }}
     - retry:
         attempts: 5
         interval: 30