From 8ce0f5b7bed3c0b62a7f7ed29d34179571f32d27 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 26 Jan 2022 08:31:37 -0500 Subject: [PATCH 1/8] log removal of root cron --- setup/so-functions | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 589427f56..d518a347a 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2049,9 +2049,6 @@ reserve_ports() { reinstall_init() { info "Putting system in state to run setup again" - - # remove all of root's cronjobs - crontab -r -u root if [[ $install_type =~ ^(MANAGER|EVAL|HELIXSENSOR|MANAGERSEARCH|STANDALONE|FLEET|IMPORT)$ ]]; then local salt_services=( "salt-master" "salt-minion" ) @@ -2062,6 +2059,9 @@ reinstall_init() { local service_retry_count=20 { + # remove all of root's cronjobs + crontab -r -u root + if command -v salt-call &> /dev/null && grep -q "master:" /etc/salt/minion 2> /dev/null; then # Disable schedule so highstate doesn't start running during the install salt-call -l info schedule.disable --local From 8aa002b82e8711d3e1f38e9eff77f67ed6385688 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 26 Jan 2022 09:33:19 -0500 Subject: [PATCH 2/8] add states to remove ca and ssl keys and certs and call them during reinstall. --- salt/ca/remove.sls | 22 +++++++ salt/influxdb/init.sls | 2 + salt/ssl/remove.sls | 134 +++++++++++++++++++++++++++++++++++++++++ setup/so-functions | 8 ++- 4 files changed, 165 insertions(+), 1 deletion(-) create mode 100644 salt/ca/remove.sls create mode 100644 salt/ssl/remove.sls diff --git a/salt/ca/remove.sls b/salt/ca/remove.sls new file mode 100644 index 000000000..aad7b4d4f --- /dev/null +++ b/salt/ca/remove.sls @@ -0,0 +1,22 @@ +{% if grains.role in ['so-helix', 'so-eval', 'so-manager', 'so-standalone', 'so-managersearch', 'so-import' ] %} + +pki_private_key: + file.absent: + - name: /etc/pki/ca.key + +pki_public_ca_crt: + file.absent: + - name: /etc/pki/ca.crt + +{% else %} + +not_a_ca: + test.succeed_without_changes: + - name: no_ca + - comment: "Not a CA, so no CA to remove." + +{% endif %} + +remove_ca-certificates.crt: + file.absent: + - name: /etc/ssl/certs/ca-certificates.crt diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index 218d2d18e..f2bdc1a1a 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -84,6 +84,8 @@ wait_for_influxdb: - retry: attempts: 5 interval: 60 + - require: + - docker_container: so-influxdb telegraf_database: influxdb_database.present: diff --git a/salt/ssl/remove.sls b/salt/ssl/remove.sls new file mode 100644 index 000000000..87e10a4ed --- /dev/null +++ b/salt/ssl/remove.sls @@ -0,0 +1,134 @@ +# Trust the CA +trusttheca: + file.absent: + - name: /etc/ssl/certs/intca.crt + +removefbcertdir: + file.absent: + - name: /etc/pki/filebeat.crt + - onlyif: "[ -d /etc/pki/filebeat.crt ]" + +removefbp8dir: + file.absent: + - name: /etc/pki/filebeat.p8 + - onlyif: "[ -d /etc/pki/filebeat.p8 ]" + +removeesp12dir: + file.absent: + - name: /etc/pki/elasticsearch.p12 + - onlyif: "[ -d /etc/pki/elasticsearch.p12 ]" + +influxdb_key: + file.absent: + - name: /etc/pki/influxdb.key + +influxdb_crt: + file.absent: + - name: /etc/pki/influxdb.crt + +{% if grains['role'] in ['so-manager', 'so-eval', 'so-helix', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode', 'so-fleet', 'so-receiver'] %} +redis_key: + file.absent: + - name: /etc/pki/redis.key + +redis_crt: + file.absent: + - name: /etc/pki/redis.crt +{% endif %} + +{% if grains['role'] in ['so-manager', 'so-eval', 'so-helix', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode', 'so-receiver'] %} +etc_filebeat_key: + file.absent: + - name: /etc/pki/filebeat.key + +etc_filebeat_crt: + file.absent: + - name: /etc/pki/filebeat.crt + + {% if grains.role not in ['so-heavynode', 'so-receiver'] %} +filebeatdir: + file.absent: + - name: /opt/so/saltstack/local/salt/filebeat/files + +registry_key: + file.absent: + - name: /etc/pki/registry.key + +registry_crt: + file.absent: + - name: /etc/pki/registry.crt + + {% endif %} + + {% if grains.role not in ['so-receiver'] %} +/etc/pki/elasticsearch.key: + file.absent: [] + +/etc/pki/elasticsearch.crt: + file.absent: [] + +remove_elasticsearch.p12: + file.absent: + - name: /etc/pki/elasticsearch.p12 + +managerssl_key: + file.absent: + - name: /etc/pki/managerssl.key + +managerssl_crt: + file.absent: + - name: /etc/pki/managerssl.crt + + {% endif %} + +fleet_key: + file.absent: + - name: /etc/pki/fleet.key + +fleet_crt: + file.absent: + - name: /etc/pki/fleet.crt + +{% endif %} + +{% if grains['role'] in ['so-sensor', 'so-manager', 'so-node', 'so-eval', 'so-helix', 'so-managersearch', 'so-heavynode', 'so-fleet', 'so-standalone', 'so-import', 'so-receiver'] %} + +fbcertdir: + file.absent: + - name: /opt/so/conf/filebeat/etc/pki + +{% endif %} + +{% if grains['role'] == 'so-fleet' %} + +managerssl_key: + file.absent: + - name: /etc/pki/managerssl.key + +managerssl_crt: + file.absent: + - name: /etc/pki/managerssl.crt + +fleet_key: + file.absent: + - name: /etc/pki/fleet.key + +fleet_crt: + file.absent: + - name: /etc/pki/fleet.crt + +{% endif %} + +{% if grains['role'] == 'so-node' %} + +/etc/pki/elasticsearch.key: + file.absent: [] + +/etc/pki/elasticsearch.crt: + file.absent: [] + +remove_elastic.p12: + file.absent: + - name: /etc/pki/elasticsearch.p12 + +{%- endif %} diff --git a/setup/so-functions b/setup/so-functions index d518a347a..8b0bf8119 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2060,7 +2060,7 @@ reinstall_init() { { # remove all of root's cronjobs - crontab -r -u root + logCmd "crontab -r -u root" if command -v salt-call &> /dev/null && grep -q "master:" /etc/salt/minion 2> /dev/null; then # Disable schedule so highstate doesn't start running during the install @@ -2091,6 +2091,9 @@ reinstall_init() { done done + logCmd "salt-call state.apply ca.remove -linfo --local" + logCmd "salt-call state.apply ssl.remove -linfo --local" + # Remove all salt configs rm -rf /etc/salt/engines/* /etc/salt/grains /etc/salt/master /etc/salt/master.d/* /etc/salt/minion /etc/salt/minion.d/* /etc/salt/pki/* /etc/salt/proxy /etc/salt/proxy.d/* /var/cache/salt/ @@ -2122,10 +2125,13 @@ reinstall_init() { remove_package launcher-final if [[ $OS == 'ubuntu' ]]; then + info "Unholding previously held packages." apt-mark unhold $(apt-mark showhold) fi } >> "$setup_log" 2>&1 + + info "System reinstall init has been completed." } reset_proxy() { From a43fb293fc6dba64dbc08beece3ed093c147aa30 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 26 Jan 2022 10:26:52 -0500 Subject: [PATCH 3/8] remove role logic --- salt/ca/remove.sls | 11 ----------- salt/ssl/remove.sls | 39 --------------------------------------- setup/so-functions | 4 ++-- 3 files changed, 2 insertions(+), 52 deletions(-) diff --git a/salt/ca/remove.sls b/salt/ca/remove.sls index aad7b4d4f..0fd8cbd9f 100644 --- a/salt/ca/remove.sls +++ b/salt/ca/remove.sls @@ -1,5 +1,3 @@ -{% if grains.role in ['so-helix', 'so-eval', 'so-manager', 'so-standalone', 'so-managersearch', 'so-import' ] %} - pki_private_key: file.absent: - name: /etc/pki/ca.key @@ -8,15 +6,6 @@ pki_public_ca_crt: file.absent: - name: /etc/pki/ca.crt -{% else %} - -not_a_ca: - test.succeed_without_changes: - - name: no_ca - - comment: "Not a CA, so no CA to remove." - -{% endif %} - remove_ca-certificates.crt: file.absent: - name: /etc/ssl/certs/ca-certificates.crt diff --git a/salt/ssl/remove.sls b/salt/ssl/remove.sls index 87e10a4ed..1e84ac5cd 100644 --- a/salt/ssl/remove.sls +++ b/salt/ssl/remove.sls @@ -1,23 +1,7 @@ -# Trust the CA trusttheca: file.absent: - name: /etc/ssl/certs/intca.crt -removefbcertdir: - file.absent: - - name: /etc/pki/filebeat.crt - - onlyif: "[ -d /etc/pki/filebeat.crt ]" - -removefbp8dir: - file.absent: - - name: /etc/pki/filebeat.p8 - - onlyif: "[ -d /etc/pki/filebeat.p8 ]" - -removeesp12dir: - file.absent: - - name: /etc/pki/elasticsearch.p12 - - onlyif: "[ -d /etc/pki/elasticsearch.p12 ]" - influxdb_key: file.absent: - name: /etc/pki/influxdb.key @@ -26,7 +10,6 @@ influxdb_crt: file.absent: - name: /etc/pki/influxdb.crt -{% if grains['role'] in ['so-manager', 'so-eval', 'so-helix', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode', 'so-fleet', 'so-receiver'] %} redis_key: file.absent: - name: /etc/pki/redis.key @@ -34,9 +17,7 @@ redis_key: redis_crt: file.absent: - name: /etc/pki/redis.crt -{% endif %} -{% if grains['role'] in ['so-manager', 'so-eval', 'so-helix', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode', 'so-receiver'] %} etc_filebeat_key: file.absent: - name: /etc/pki/filebeat.key @@ -45,7 +26,6 @@ etc_filebeat_crt: file.absent: - name: /etc/pki/filebeat.crt - {% if grains.role not in ['so-heavynode', 'so-receiver'] %} filebeatdir: file.absent: - name: /opt/so/saltstack/local/salt/filebeat/files @@ -58,9 +38,6 @@ registry_crt: file.absent: - name: /etc/pki/registry.crt - {% endif %} - - {% if grains.role not in ['so-receiver'] %} /etc/pki/elasticsearch.key: file.absent: [] @@ -79,8 +56,6 @@ managerssl_crt: file.absent: - name: /etc/pki/managerssl.crt - {% endif %} - fleet_key: file.absent: - name: /etc/pki/fleet.key @@ -89,17 +64,9 @@ fleet_crt: file.absent: - name: /etc/pki/fleet.crt -{% endif %} - -{% if grains['role'] in ['so-sensor', 'so-manager', 'so-node', 'so-eval', 'so-helix', 'so-managersearch', 'so-heavynode', 'so-fleet', 'so-standalone', 'so-import', 'so-receiver'] %} - fbcertdir: file.absent: - name: /opt/so/conf/filebeat/etc/pki - -{% endif %} - -{% if grains['role'] == 'so-fleet' %} managerssl_key: file.absent: @@ -117,10 +84,6 @@ fleet_crt: file.absent: - name: /etc/pki/fleet.crt -{% endif %} - -{% if grains['role'] == 'so-node' %} - /etc/pki/elasticsearch.key: file.absent: [] @@ -130,5 +93,3 @@ fleet_crt: remove_elastic.p12: file.absent: - name: /etc/pki/elasticsearch.p12 - -{%- endif %} diff --git a/setup/so-functions b/setup/so-functions index 8b0bf8119..996620336 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2091,8 +2091,8 @@ reinstall_init() { done done - logCmd "salt-call state.apply ca.remove -linfo --local" - logCmd "salt-call state.apply ssl.remove -linfo --local" + logCmd "salt-call state.apply ca.remove -linfo --local --file-root=../salt" + logCmd "salt-call state.apply ssl.remove -linfo --local --file-root=../salt" # Remove all salt configs rm -rf /etc/salt/engines/* /etc/salt/grains /etc/salt/master /etc/salt/master.d/* /etc/salt/minion /etc/salt/minion.d/* /etc/salt/pki/* /etc/salt/proxy /etc/salt/proxy.d/* /var/cache/salt/ From a0e493a186b89ef3318c03d4f61e282dcaf9fb3e Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 26 Jan 2022 10:50:35 -0500 Subject: [PATCH 4/8] remove dupe ids --- salt/ssl/remove.sls | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/salt/ssl/remove.sls b/salt/ssl/remove.sls index 1e84ac5cd..276ecbe37 100644 --- a/salt/ssl/remove.sls +++ b/salt/ssl/remove.sls @@ -68,14 +68,6 @@ fbcertdir: file.absent: - name: /opt/so/conf/filebeat/etc/pki -managerssl_key: - file.absent: - - name: /etc/pki/managerssl.key - -managerssl_crt: - file.absent: - - name: /etc/pki/managerssl.crt - fleet_key: file.absent: - name: /etc/pki/fleet.key @@ -83,13 +75,3 @@ fleet_key: fleet_crt: file.absent: - name: /etc/pki/fleet.crt - -/etc/pki/elasticsearch.key: - file.absent: [] - -/etc/pki/elasticsearch.crt: - file.absent: [] - -remove_elastic.p12: - file.absent: - - name: /etc/pki/elasticsearch.p12 From 1337af9d6954e1ea1f42e96837b0d3fb16fd351f Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 26 Jan 2022 11:07:06 -0500 Subject: [PATCH 5/8] more dupes --- salt/ssl/remove.sls | 8 -------- 1 file changed, 8 deletions(-) diff --git a/salt/ssl/remove.sls b/salt/ssl/remove.sls index 276ecbe37..4eb0eb442 100644 --- a/salt/ssl/remove.sls +++ b/salt/ssl/remove.sls @@ -67,11 +67,3 @@ fleet_crt: fbcertdir: file.absent: - name: /opt/so/conf/filebeat/etc/pki - -fleet_key: - file.absent: - - name: /etc/pki/fleet.key - -fleet_crt: - file.absent: - - name: /etc/pki/fleet.crt From 2aa19b78da71ad81ec353f97fc02d99998d2eeeb Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 26 Jan 2022 11:27:35 -0500 Subject: [PATCH 6/8] dont remove ca-certificates.crt --- salt/ca/remove.sls | 4 ---- 1 file changed, 4 deletions(-) diff --git a/salt/ca/remove.sls b/salt/ca/remove.sls index 0fd8cbd9f..3af355951 100644 --- a/salt/ca/remove.sls +++ b/salt/ca/remove.sls @@ -5,7 +5,3 @@ pki_private_key: pki_public_ca_crt: file.absent: - name: /etc/pki/ca.crt - -remove_ca-certificates.crt: - file.absent: - - name: /etc/ssl/certs/ca-certificates.crt From e63f35a223d37802318fc0c8436c13c934454b08 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 27 Jan 2022 15:19:33 -0500 Subject: [PATCH 7/8] change to test --- salt/ssl/init.sls | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index 57a8b519b..0958c0db1 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -56,17 +56,17 @@ m2cryptopkgs: removefbcertdir: file.absent: - name: /etc/pki/filebeat.crt - - onlyif: "[ -d /etc/pki/filebeat.crt ]" + - onlyif: "test -d /etc/pki/filebeat.crt" removefbp8dir: file.absent: - name: /etc/pki/filebeat.p8 - - onlyif: "[ -d /etc/pki/filebeat.p8 ]" + - onlyif: "test -d /etc/pki/filebeat.p8" removeesp12dir: file.absent: - name: /etc/pki/elasticsearch.p12 - - onlyif: "[ -d /etc/pki/elasticsearch.p12 ]" + - onlyif: "test -d /etc/pki/elasticsearch.p12" influxdb_key: x509.private_key_managed: From 6cc8e4355e7ba015aef269753ecde46217444d75 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 27 Jan 2022 15:31:42 -0500 Subject: [PATCH 8/8] exclude salt ERROR seen during reinstall --- setup/so-setup | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index 0d8d6a471..90e57646d 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -968,10 +968,11 @@ if [[ $success != 0 ]]; then SO_ERROR=1; fi # Check entire setup log for errors or unexpected salt states and ensure cron jobs are not reporting errors to root's mailbox # Ignore "Status .* was not found" due to output from salt http.query or http.wait_for_successful_query states used with retry -if grep -E "ERROR|Result: False" $setup_log | grep -qvE "Status .* was not found" || [[ -s /var/spool/mail/root && "$setup_type" == "iso" ]]; then +# Uncaught exception, closing connection|Exception in callback None - this is seen during influxdb / http.wait_for_successful_query state for ubuntu reinstall +if grep -E "ERROR|Result: False" $setup_log | grep -qvE "Status .* was not found|An exception occurred in this state|Uncaught exception, closing connection|Exception in callback None" || [[ -s /var/spool/mail/root && "$setup_type" == "iso" ]]; then SO_ERROR=1 - grep --color=never "ERROR" "$setup_log" | grep -qvE "Status .* was not found" > "$error_log" + grep --color=never "ERROR" "$setup_log" | grep -qvE "Status .* was not found|An exception occurred in this state|Uncaught exception, closing connection|Exception in callback None" > "$error_log" fi if [[ -n $SO_ERROR ]]; then