From 138aa9c5542e3fb3a5a85c1a1edc66d6d13ae8ff Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 18 Oct 2023 13:54:14 -0400 Subject: [PATCH 01/12] update the mine with the ca when it is created or changed --- salt/ca/init.sls | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/salt/ca/init.sls b/salt/ca/init.sls index 0eaf86b3c..2f15872ea 100644 --- a/salt/ca/init.sls +++ b/salt/ca/init.sls @@ -50,6 +50,12 @@ pki_public_ca_crt: attempts: 5 interval: 30 +mine_update_ca_crt: + - module.run: + - mine.update: [] + - onchanges: + - file: pki_public_ca_crt + cakeyperms: file.managed: - replace: False From 8e68f9631618c58f9f95d507ebfcc888b143532f Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 18 Oct 2023 13:59:15 -0400 Subject: [PATCH 02/12] check that the manager has a ca in the mine and that it is valid --- salt/salt/engines/master/checkmine.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/salt/salt/engines/master/checkmine.py b/salt/salt/engines/master/checkmine.py index c62c1f058..e72d2fbe6 100644 --- a/salt/salt/engines/master/checkmine.py +++ b/salt/salt/engines/master/checkmine.py @@ -46,6 +46,25 @@ def start(interval=60): mine_update(minion) continue + # if a manager check that the ca in in the mine and it is correct + if minion.split('_')[-1] in ['manager', 'managersearch', 'eval', 'standalone', 'import']: + x509 = __salt__['saltutil.runner']('mine.get', tgt=minion, fun='x509.get_pem_entries') + try: + ca_crt = x509[minion]['/etc/pki/ca.crt'] + log.debug('checkmine engine: found minion %s has ca_crt: %s' % (minion, ca_crt)) + # since the cert is defined, make sure it is valid + if not __salt__['x509.verify_private_key'](private_key='/etc/pki/ca.key', public_key='/etc/pki/ca.crt'): + log.error('checkmine engine: found minion %s does\'t have a valid ca_crt in the mine' % (minion)) + log.error('checkmine engine: %s: ca_crt: %s' % (minion, ca_crt)) + mine_delete(minion, 'x509.get_pem_entries') + mine_update(minion) + else: + log.debug('checkmine engine: found minion %s has a valid ca_crt in the mine' % (minion)) + except IndexError: + log.error('checkmine engine: found minion %s does\'t have a ca_crt in the mine' % (minion)) + mine_delete(minion, 'x509.get_pem_entries') + mine_update(minion) + # Update the mine if the ip in the mine doesn't match returned from manage.alived network_ip_addrs = __salt__['saltutil.runner']('mine.get', tgt=minion, fun='network.ip_addrs') try: From c3cde61202507f177f30212fa75d8b258e6da671 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 18 Oct 2023 15:01:26 -0400 Subject: [PATCH 03/12] docker service watches and requires the intca --- salt/docker/init.sls | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/salt/docker/init.sls b/salt/docker/init.sls index 45ba4a1ac..769c58af8 100644 --- a/salt/docker/init.sls +++ b/salt/docker/init.sls @@ -6,6 +6,9 @@ {% from 'docker/docker.map.jinja' import DOCKER %} {% from 'vars/globals.map.jinja' import GLOBALS %} +# include ssl since docker service requires the intca +include: + - ssl dockergroup: group.present: @@ -86,6 +89,11 @@ docker_running: - enable: True - watch: - file: docker_daemon + - x509: trusttheca + - require: + - file: docker_daemon + - x509: trusttheca + # Reserve OS ports for Docker proxy in case boot settings are not already applied/present # 57314 = Strelka, 47760-47860 = Zeek From 1999db0bb3d593df97a1267e398813d4fabcd084 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 18 Oct 2023 15:02:22 -0400 Subject: [PATCH 04/12] apply ca state early in setup --- setup/so-setup | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/setup/so-setup b/setup/so-setup index 4db24aa1a..052111591 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -714,12 +714,19 @@ if ! [[ -f $install_opt_file ]]; then logCmd "salt-call state.apply common.packages" logCmd "salt-call state.apply common" + # apply the ca state to create the ca and put it in the mine early in the install + # this is done before the salt.master state puts the checkmine engine in place and starts + # checking for a valid ca in the mine for the manager + # the minion ip will already be in the mine from configure_minion function in so-functions + logCmd "salt-call state.apply ca" + # this will apply the salt.minion state first since salt.master includes salt.minion logCmd "salt-call state.apply salt.master" - # wait here until we get a response from the salt-master since it may have just restarted # exit setup after 5-6 minutes of trying check_salt_master_status || fail "Can't access salt master or it is not ready" + # this will also call the ssl state since docker requires the intca + # the salt-minion service will need to be up on the manager to sign requests logCmd "salt-call state.apply docker" firewall_generate_templates set_initial_firewall_policy From 2206cdb0fad836de3ab5a0c9197416e1ed9eb537 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 18 Oct 2023 15:04:39 -0400 Subject: [PATCH 05/12] change soup comment --- pillar/nodegroups/init.sls | 0 salt/manager/tools/sbin/soup | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 pillar/nodegroups/init.sls diff --git a/pillar/nodegroups/init.sls b/pillar/nodegroups/init.sls deleted file mode 100644 index e69de29bb..000000000 diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 84501bad5..fc07765b8 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -578,7 +578,7 @@ update_centos_repo() { } update_salt_mine() { - echo "Populating the mine with network.ip_addrs pillar.host.mainint for each host." + echo "Populating the mine with mine_functions for each host." set +e salt \* mine.update -b 50 set -e From 1c1b23c328af4d2788992c1cfc87451d97e6648b Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 18 Oct 2023 15:07:18 -0400 Subject: [PATCH 06/12] fix mine update for ca --- salt/ca/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/ca/init.sls b/salt/ca/init.sls index 2f15872ea..477b51a5d 100644 --- a/salt/ca/init.sls +++ b/salt/ca/init.sls @@ -51,7 +51,7 @@ pki_public_ca_crt: interval: 30 mine_update_ca_crt: - - module.run: + module.run: - mine.update: [] - onchanges: - file: pki_public_ca_crt From e58c1e189c63bd6aab9d6b7a32b06ee6cc3a075b Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 18 Oct 2023 15:10:17 -0400 Subject: [PATCH 07/12] use x509 instead of file for onchanges --- salt/ca/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/ca/init.sls b/salt/ca/init.sls index 477b51a5d..895e8235a 100644 --- a/salt/ca/init.sls +++ b/salt/ca/init.sls @@ -54,7 +54,7 @@ mine_update_ca_crt: module.run: - mine.update: [] - onchanges: - - file: pki_public_ca_crt + - x509: pki_public_ca_crt cakeyperms: file.managed: From ac28e1b967f35c1f3f21424b14036367b627f48c Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 18 Oct 2023 15:53:12 -0400 Subject: [PATCH 08/12] verify crt and key differently in checkmine --- salt/salt/engines/master/checkmine.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/salt/salt/engines/master/checkmine.py b/salt/salt/engines/master/checkmine.py index e72d2fbe6..1440fb72f 100644 --- a/salt/salt/engines/master/checkmine.py +++ b/salt/salt/engines/master/checkmine.py @@ -53,17 +53,20 @@ def start(interval=60): ca_crt = x509[minion]['/etc/pki/ca.crt'] log.debug('checkmine engine: found minion %s has ca_crt: %s' % (minion, ca_crt)) # since the cert is defined, make sure it is valid - if not __salt__['x509.verify_private_key'](private_key='/etc/pki/ca.key', public_key='/etc/pki/ca.crt'): + import salt.modules.x509_v2 as x509_v2 + if not x509_v2.verify_private_key('/etc/pki/ca.key', '/etc/pki/ca.crt'): log.error('checkmine engine: found minion %s does\'t have a valid ca_crt in the mine' % (minion)) log.error('checkmine engine: %s: ca_crt: %s' % (minion, ca_crt)) mine_delete(minion, 'x509.get_pem_entries') mine_update(minion) + continue else: log.debug('checkmine engine: found minion %s has a valid ca_crt in the mine' % (minion)) except IndexError: log.error('checkmine engine: found minion %s does\'t have a ca_crt in the mine' % (minion)) mine_delete(minion, 'x509.get_pem_entries') mine_update(minion) + continue # Update the mine if the ip in the mine doesn't match returned from manage.alived network_ip_addrs = __salt__['saltutil.runner']('mine.get', tgt=minion, fun='network.ip_addrs') From a52ee063e597b010df8996fd02423598570fd9a6 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 18 Oct 2023 16:35:33 -0400 Subject: [PATCH 09/12] use generate_ca and generate_ssl functions and move them up --- setup/so-setup | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index 052111591..73ef29f4b 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -718,7 +718,7 @@ if ! [[ -f $install_opt_file ]]; then # this is done before the salt.master state puts the checkmine engine in place and starts # checking for a valid ca in the mine for the manager # the minion ip will already be in the mine from configure_minion function in so-functions - logCmd "salt-call state.apply ca" + generate_ca # this will apply the salt.minion state first since salt.master includes salt.minion logCmd "salt-call state.apply salt.master" # wait here until we get a response from the salt-master since it may have just restarted @@ -727,6 +727,7 @@ if ! [[ -f $install_opt_file ]]; then # this will also call the ssl state since docker requires the intca # the salt-minion service will need to be up on the manager to sign requests + generate_ssl logCmd "salt-call state.apply docker" firewall_generate_templates set_initial_firewall_policy @@ -734,8 +735,6 @@ if ! [[ -f $install_opt_file ]]; then title "Downloading Elastic Agent Artifacts" download_elastic_agent_artifacts - generate_ca - generate_ssl logCmd "salt-call state.apply -l info firewall" # create these so the registry state can add so-registry to /opt/so/conf/so-status/so-status.conf From 9eb682bc40fca2463275e43e2db37a7a52978216 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 18 Oct 2023 18:37:35 -0400 Subject: [PATCH 10/12] generate_ca after salt-master and salt-minion states run --- salt/common/tools/sbin/so-common | 2 +- setup/so-functions | 10 ++++++++++ setup/so-setup | 9 +++------ 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index c19d51a42..be001f0d7 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -153,7 +153,7 @@ check_salt_master_status() { } check_salt_minion_status() { - local timeout=$1 + local timeout="${1:-5}" echo "Checking if the salt minion will respond to jobs" >> "$setup_log" 2>&1 salt "$MINION_ID" test.ping -t $timeout > /dev/null 2>&1 local status=$? diff --git a/setup/so-functions b/setup/so-functions index 68fd01550..7ead07ca7 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2495,6 +2495,16 @@ wait_for_file() { wait_for_salt_minion() { retry 60 5 "journalctl -u salt-minion.service | grep 'Minion is ready to receive requests'" >> "$setup_log" 2>&1 || fail_setup + local attempt=0 + # each attempts would take about 15 seconds + local maxAttempts=20 + until check_salt_minion_status; do + attempt=$((attempt+1)) + if [[ $attempt -gt $maxAttempts ]]; then + fail_setup + fi + sleep 10 + done } verify_setup() { diff --git a/setup/so-setup b/setup/so-setup index 73ef29f4b..543ac0156 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -714,17 +714,14 @@ if ! [[ -f $install_opt_file ]]; then logCmd "salt-call state.apply common.packages" logCmd "salt-call state.apply common" - # apply the ca state to create the ca and put it in the mine early in the install - # this is done before the salt.master state puts the checkmine engine in place and starts - # checking for a valid ca in the mine for the manager - # the minion ip will already be in the mine from configure_minion function in so-functions - generate_ca # this will apply the salt.minion state first since salt.master includes salt.minion logCmd "salt-call state.apply salt.master" # wait here until we get a response from the salt-master since it may have just restarted # exit setup after 5-6 minutes of trying check_salt_master_status || fail "Can't access salt master or it is not ready" - + # apply the ca state to create the ca and put it in the mine early in the install + # the minion ip will already be in the mine from configure_minion function in so-functions + generate_ca # this will also call the ssl state since docker requires the intca # the salt-minion service will need to be up on the manager to sign requests generate_ssl From ff18b1f074c8448b9503e04acf7b6df41240fde5 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 18 Oct 2023 18:45:14 -0400 Subject: [PATCH 11/12] remove redirect --- salt/common/tools/sbin/so-common | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index be001f0d7..1a5e5df6c 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -154,13 +154,13 @@ check_salt_master_status() { check_salt_minion_status() { local timeout="${1:-5}" - echo "Checking if the salt minion will respond to jobs" >> "$setup_log" 2>&1 + echo "Checking if the salt minion will respond to jobs" salt "$MINION_ID" test.ping -t $timeout > /dev/null 2>&1 local status=$? if [ $status -gt 0 ]; then - echo " Minion did not respond" >> "$setup_log" 2>&1 + echo " Minion did not respond" else - echo " Received job response from salt minion" >> "$setup_log" 2>&1 + echo " Received job response from salt minion" fi return $status From f30a652e19ea9c363f5bf4bba682153c2f56fc5b Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 18 Oct 2023 19:31:45 -0400 Subject: [PATCH 12/12] add back redirects --- salt/common/tools/sbin/so-common | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 1a5e5df6c..dce86362f 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -154,13 +154,13 @@ check_salt_master_status() { check_salt_minion_status() { local timeout="${1:-5}" - echo "Checking if the salt minion will respond to jobs" + echo "Checking if the salt minion will respond to jobs" >> "$setup_log" 2>&1 salt "$MINION_ID" test.ping -t $timeout > /dev/null 2>&1 local status=$? if [ $status -gt 0 ]; then - echo " Minion did not respond" + echo " Minion did not respond" >> "$setup_log" 2>&1 else - echo " Received job response from salt minion" + echo " Received job response from salt minion" >> "$setup_log" 2>&1 fi return $status