From 2e16250c93d000c20e451ed0e0b539e6c8f84bfa Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 20 Oct 2023 10:00:39 -0400 Subject: [PATCH 01/10] handle a minion not being in the mine data return --- salt/salt/engines/master/checkmine.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/salt/salt/engines/master/checkmine.py b/salt/salt/engines/master/checkmine.py index 1440fb72f..afeb92536 100644 --- a/salt/salt/engines/master/checkmine.py +++ b/salt/salt/engines/master/checkmine.py @@ -67,6 +67,10 @@ def start(interval=60): mine_delete(minion, 'x509.get_pem_entries') mine_update(minion) continue + except KeyError: + log.error('checkmine engine: found minion %s is not in the mine' % (minion)) + mine_flush(minion) + mine_update(minion) # Update the mine if the ip in the mine doesn't match returned from manage.alived network_ip_addrs = __salt__['saltutil.runner']('mine.get', tgt=minion, fun='network.ip_addrs') @@ -77,5 +81,8 @@ def start(interval=60): log.error('checkmine engine: found minion %s does\'t have a mine_ip' % (minion)) mine_delete(minion, 'network.ip_addrs') mine_update(minion) - + except KeyError: + log.error('checkmine engine: found minion %s is not in the mine' % (minion)) + mine_flush(minion) + mine_update(minion) sleep(interval) From ef2b89f5bf37c304ab431aa0fbeb21b3f8a3ecd7 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 20 Oct 2023 13:40:40 -0400 Subject: [PATCH 02/10] fix attempts logic --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 7ead07ca7..1c9b0f43d 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2500,7 +2500,7 @@ wait_for_salt_minion() { local maxAttempts=20 until check_salt_minion_status; do attempt=$((attempt+1)) - if [[ $attempt -gt $maxAttempts ]]; then + if [[ $attempt -eq $maxAttempts ]]; then fail_setup fi sleep 10 From 99662c999fc5b437dc9eba8061a4c874e1692866 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 20 Oct 2023 13:41:24 -0400 Subject: [PATCH 03/10] log operation and minion target --- salt/soc/files/bin/salt-relay.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/soc/files/bin/salt-relay.sh b/salt/soc/files/bin/salt-relay.sh index ac56ec6e3..fea81728d 100755 --- a/salt/soc/files/bin/salt-relay.sh +++ b/salt/soc/files/bin/salt-relay.sh @@ -67,10 +67,10 @@ function manage_minion() { response=$(so-minion "-o=$op" "-m=$minion_id") exit_code=$? if [[ exit_code -eq 0 ]]; then - log "Successful command execution" + log "Successful '$op' command execution on $minion_id" respond "$id" "true" else - log "Unsuccessful command execution: $response ($exit_code)" + log "Unsuccessful '$op' command execution on $minion_id: $response ($exit_code)" respond "$id" "false" fi } From 6d77b1e4c389a90ae475e2e523e302572fdf244b Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 20 Oct 2023 13:41:53 -0400 Subject: [PATCH 04/10] continue loop if minion not in mine --- salt/salt/engines/master/checkmine.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/salt/engines/master/checkmine.py b/salt/salt/engines/master/checkmine.py index afeb92536..f33392575 100644 --- a/salt/salt/engines/master/checkmine.py +++ b/salt/salt/engines/master/checkmine.py @@ -71,6 +71,7 @@ def start(interval=60): log.error('checkmine engine: found minion %s is not in the mine' % (minion)) mine_flush(minion) mine_update(minion) + continue # Update the mine if the ip in the mine doesn't match returned from manage.alived network_ip_addrs = __salt__['saltutil.runner']('mine.get', tgt=minion, fun='network.ip_addrs') @@ -85,4 +86,5 @@ def start(interval=60): log.error('checkmine engine: found minion %s is not in the mine' % (minion)) mine_flush(minion) mine_update(minion) + sleep(interval) From c588bf4395aa3d6734c2787695a9b786ceb69d30 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 20 Oct 2023 13:43:12 -0400 Subject: [PATCH 05/10] update mine and highstate minion when added --- salt/manager/tools/sbin/so-minion | 35 ++++++++++++++++++++++++------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index 64084dbd0..325bedf4a 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -407,11 +407,9 @@ function update_logstash_outputs() { } function updateMine() { - salt "$MINION_ID" mine.send network.ip_addrs interface="$MNIC" -} -function apply_ES_state() { - salt-call state.apply elasticsearch concurrent=True + retry 10 2 "salt '$MINION_ID' mine.update" True } + function createEVAL() { is_pcaplimit=true add_elasticsearch_to_minion @@ -547,8 +545,6 @@ function createSEARCHNODE() { add_elasticsearch_to_minion add_logstash_to_minion add_telegraf_to_minion - updateMine - apply_ES_state } function createRECEIVER() { @@ -563,6 +559,19 @@ function createDESKTOP() { } function testConnection() { + # the minion should be trying to auth every 10 seconds so 15 seconds should be more than enough time to see this in the log + # this retry was put in because it is possible that a minion is attempted to be pinged before it has authenticated and connected to the Salt master + # causing the first ping to fail and typically wouldn't be successful until the second ping + # this check may pass without the minion being authenticated if it was previously connected and the line exists in the log + retry 15 1 "grep 'Authentication accepted from $MINION_ID' /opt/so/log/salt/master" + local retauth=$? + if [[ $retauth != 0 ]]; then + echo "The Minion did not authenticate with the Salt master in the allotted time" + echo "Deleting the key" + deleteminion + exit 1 + fi + retry 15 3 "salt '$MINION_ID' test.ping" True local ret=$? if [[ $ret != 0 ]]; then @@ -582,9 +591,9 @@ if [[ "$OPERATION" = 'delete' ]]; then deleteminion fi -if [[ "$OPERATION" = 'add' || "$OPERATION" = 'setup' ]]; then +if [[ "$OPERATION" == 'add' || "$OPERATION" == 'setup' ]]; then # Skip this if its setup - if [ $OPERATION != 'setup' ]; then + if [[ $OPERATION == 'add' ]]; then # Accept the salt key acceptminion # Test to see if the minion was accepted @@ -605,8 +614,18 @@ if [[ "$OPERATION" = 'add' || "$OPERATION" = 'setup' ]]; then else add_sensoroni_to_minion fi + create$NODETYPE echo "Minion file created for $MINION_ID" + + if [[ "$OPERATION" == 'add' ]]; then + # tell the minion to populate the mine with data from mine_functions which is populated during setup + # this only needs to happen on non managers since they handle this during setup + # and they need to wait for ca creation to update the mine + updateMine + # run this async so the cli doesn't wait for a return + salt "$MINION_ID" state.highstate --async + fi fi if [[ "$OPERATION" = 'test' ]]; then From c4093394464bded7ad6ce2b76720a97e823179d3 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 20 Oct 2023 13:46:24 -0400 Subject: [PATCH 06/10] change post setup highstate cron to 5 minutes since accepting minion runs a highstate --- salt/setup/highstate_cron.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/setup/highstate_cron.sls b/salt/setup/highstate_cron.sls index 862968d97..f8f76e737 100644 --- a/salt/setup/highstate_cron.sls +++ b/salt/setup/highstate_cron.sls @@ -3,5 +3,5 @@ post_setup_cron: - name: 'PATH=$PATH:/usr/sbin salt-call state.highstate' - identifier: post_setup_cron - user: root - - minute: '*/1' + - minute: '*/5' - identifier: post_setup_cron From 7e3aa11a7341c6324b13dbf8367fe21ae80c8870 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 20 Oct 2023 16:27:20 -0400 Subject: [PATCH 07/10] check mine is populated with ip before telling node to highstate --- salt/manager/tools/sbin/so-minion | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index 325bedf4a..f92c0ff55 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -406,8 +406,14 @@ function update_logstash_outputs() { curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/outputs/so-manager_logstash" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" } +function checkMine() { + func = $1 + retry 20 1 "salt-run '$MINION_ID' mine.get '$func'" "$MINION_ID" + +} + function updateMine() { - retry 10 2 "salt '$MINION_ID' mine.update" True + retry 20 1 "salt '$MINION_ID' mine.update" True } function createEVAL() { @@ -623,6 +629,7 @@ if [[ "$OPERATION" == 'add' || "$OPERATION" == 'setup' ]]; then # this only needs to happen on non managers since they handle this during setup # and they need to wait for ca creation to update the mine updateMine + checkMine "network.ip_addrs" # run this async so the cli doesn't wait for a return salt "$MINION_ID" state.highstate --async fi From dc3ca99c12e56c756cbee4998b962dd4410c8004 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 20 Oct 2023 17:16:33 -0400 Subject: [PATCH 08/10] ask the minion if it can see itself in the mine --- salt/manager/tools/sbin/so-minion | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index f92c0ff55..eca96da5c 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -407,8 +407,9 @@ function update_logstash_outputs() { } function checkMine() { - func = $1 - retry 20 1 "salt-run '$MINION_ID' mine.get '$func'" "$MINION_ID" + local func=$1 + # make sure the minion sees itself in the mine since it needs to see itself for states as opposed to using salt-run + retry 20 1 "salt '$MINION_ID' mine.get '\*' '$func'" "$MINION_ID" } From 030a667d260ae59e849b8c59814c9f371580e458 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 23 Oct 2023 11:47:14 -0400 Subject: [PATCH 09/10] Add -watch to soctopus saltstate for file SOCtopus.conf. Makes container restart @ highstate if file is updated. Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/soctopus/enabled.sls | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/soctopus/enabled.sls b/salt/soctopus/enabled.sls index 0474998cb..567562fbb 100644 --- a/salt/soctopus/enabled.sls +++ b/salt/soctopus/enabled.sls @@ -52,6 +52,8 @@ so-soctopus: - {{ XTRAENV }} {% endfor %} {% endif %} + - watch: + - file: /opt/so/conf/soctopus/SOCtopus.conf - require: - file: soctopusconf - file: navigatordefaultlayer From 660020cc760f2a80b6175922718e35be134d5084 Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 23 Oct 2023 15:45:41 -0400 Subject: [PATCH 10/10] Parse pkt_src for Suricata logs --- salt/elasticsearch/files/ingest/suricata.common | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/elasticsearch/files/ingest/suricata.common b/salt/elasticsearch/files/ingest/suricata.common index e12fea0be..6b6a03a60 100644 --- a/salt/elasticsearch/files/ingest/suricata.common +++ b/salt/elasticsearch/files/ingest/suricata.common @@ -2,6 +2,7 @@ "description" : "suricata.common", "processors" : [ { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.pkt_src", "target_field": "network.packet_source","ignore_failure": true } }, { "rename": { "field": "message2.proto", "target_field": "network.transport", "ignore_failure": true } }, { "rename": { "field": "message2.flow_id", "target_field": "log.id.uid", "ignore_failure": true } }, { "rename": { "field": "message2.src_ip", "target_field": "source.ip", "ignore_failure": true } },