diff --git a/salt/elasticsearch/files/ingest/suricata.common b/salt/elasticsearch/files/ingest/suricata.common index e12fea0be..6b6a03a60 100644 --- a/salt/elasticsearch/files/ingest/suricata.common +++ b/salt/elasticsearch/files/ingest/suricata.common @@ -2,6 +2,7 @@ "description" : "suricata.common", "processors" : [ { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.pkt_src", "target_field": "network.packet_source","ignore_failure": true } }, { "rename": { "field": "message2.proto", "target_field": "network.transport", "ignore_failure": true } }, { "rename": { "field": "message2.flow_id", "target_field": "log.id.uid", "ignore_failure": true } }, { "rename": { "field": "message2.src_ip", "target_field": "source.ip", "ignore_failure": true } }, diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index 64084dbd0..eca96da5c 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -406,12 +406,17 @@ function update_logstash_outputs() { curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/outputs/so-manager_logstash" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" } +function checkMine() { + local func=$1 + # make sure the minion sees itself in the mine since it needs to see itself for states as opposed to using salt-run + retry 20 1 "salt '$MINION_ID' mine.get '\*' '$func'" "$MINION_ID" + +} + function updateMine() { - salt "$MINION_ID" mine.send network.ip_addrs interface="$MNIC" -} -function apply_ES_state() { - salt-call state.apply elasticsearch concurrent=True + retry 20 1 "salt '$MINION_ID' mine.update" True } + function createEVAL() { is_pcaplimit=true add_elasticsearch_to_minion @@ -547,8 +552,6 @@ function createSEARCHNODE() { add_elasticsearch_to_minion add_logstash_to_minion add_telegraf_to_minion - updateMine - apply_ES_state } function createRECEIVER() { @@ -563,6 +566,19 @@ function createDESKTOP() { } function testConnection() { + # the minion should be trying to auth every 10 seconds so 15 seconds should be more than enough time to see this in the log + # this retry was put in because it is possible that a minion is attempted to be pinged before it has authenticated and connected to the Salt master + # causing the first ping to fail and typically wouldn't be successful until the second ping + # this check may pass without the minion being authenticated if it was previously connected and the line exists in the log + retry 15 1 "grep 'Authentication accepted from $MINION_ID' /opt/so/log/salt/master" + local retauth=$? + if [[ $retauth != 0 ]]; then + echo "The Minion did not authenticate with the Salt master in the allotted time" + echo "Deleting the key" + deleteminion + exit 1 + fi + retry 15 3 "salt '$MINION_ID' test.ping" True local ret=$? if [[ $ret != 0 ]]; then @@ -582,9 +598,9 @@ if [[ "$OPERATION" = 'delete' ]]; then deleteminion fi -if [[ "$OPERATION" = 'add' || "$OPERATION" = 'setup' ]]; then +if [[ "$OPERATION" == 'add' || "$OPERATION" == 'setup' ]]; then # Skip this if its setup - if [ $OPERATION != 'setup' ]; then + if [[ $OPERATION == 'add' ]]; then # Accept the salt key acceptminion # Test to see if the minion was accepted @@ -605,8 +621,19 @@ if [[ "$OPERATION" = 'add' || "$OPERATION" = 'setup' ]]; then else add_sensoroni_to_minion fi + create$NODETYPE echo "Minion file created for $MINION_ID" + + if [[ "$OPERATION" == 'add' ]]; then + # tell the minion to populate the mine with data from mine_functions which is populated during setup + # this only needs to happen on non managers since they handle this during setup + # and they need to wait for ca creation to update the mine + updateMine + checkMine "network.ip_addrs" + # run this async so the cli doesn't wait for a return + salt "$MINION_ID" state.highstate --async + fi fi if [[ "$OPERATION" = 'test' ]]; then diff --git a/salt/salt/engines/master/checkmine.py b/salt/salt/engines/master/checkmine.py index 1440fb72f..f33392575 100644 --- a/salt/salt/engines/master/checkmine.py +++ b/salt/salt/engines/master/checkmine.py @@ -67,6 +67,11 @@ def start(interval=60): mine_delete(minion, 'x509.get_pem_entries') mine_update(minion) continue + except KeyError: + log.error('checkmine engine: found minion %s is not in the mine' % (minion)) + mine_flush(minion) + mine_update(minion) + continue # Update the mine if the ip in the mine doesn't match returned from manage.alived network_ip_addrs = __salt__['saltutil.runner']('mine.get', tgt=minion, fun='network.ip_addrs') @@ -77,5 +82,9 @@ def start(interval=60): log.error('checkmine engine: found minion %s does\'t have a mine_ip' % (minion)) mine_delete(minion, 'network.ip_addrs') mine_update(minion) + except KeyError: + log.error('checkmine engine: found minion %s is not in the mine' % (minion)) + mine_flush(minion) + mine_update(minion) sleep(interval) diff --git a/salt/setup/highstate_cron.sls b/salt/setup/highstate_cron.sls index 862968d97..f8f76e737 100644 --- a/salt/setup/highstate_cron.sls +++ b/salt/setup/highstate_cron.sls @@ -3,5 +3,5 @@ post_setup_cron: - name: 'PATH=$PATH:/usr/sbin salt-call state.highstate' - identifier: post_setup_cron - user: root - - minute: '*/1' + - minute: '*/5' - identifier: post_setup_cron diff --git a/salt/soc/files/bin/salt-relay.sh b/salt/soc/files/bin/salt-relay.sh index ac56ec6e3..fea81728d 100755 --- a/salt/soc/files/bin/salt-relay.sh +++ b/salt/soc/files/bin/salt-relay.sh @@ -67,10 +67,10 @@ function manage_minion() { response=$(so-minion "-o=$op" "-m=$minion_id") exit_code=$? if [[ exit_code -eq 0 ]]; then - log "Successful command execution" + log "Successful '$op' command execution on $minion_id" respond "$id" "true" else - log "Unsuccessful command execution: $response ($exit_code)" + log "Unsuccessful '$op' command execution on $minion_id: $response ($exit_code)" respond "$id" "false" fi } diff --git a/salt/soctopus/enabled.sls b/salt/soctopus/enabled.sls index 0474998cb..567562fbb 100644 --- a/salt/soctopus/enabled.sls +++ b/salt/soctopus/enabled.sls @@ -52,6 +52,8 @@ so-soctopus: - {{ XTRAENV }} {% endfor %} {% endif %} + - watch: + - file: /opt/so/conf/soctopus/SOCtopus.conf - require: - file: soctopusconf - file: navigatordefaultlayer diff --git a/setup/so-functions b/setup/so-functions index 546904d03..b64daaa92 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2494,6 +2494,20 @@ wait_for_file() { return 1 } +wait_for_salt_minion() { + retry 60 5 "journalctl -u salt-minion.service | grep 'Minion is ready to receive requests'" >> "$setup_log" 2>&1 || fail_setup + local attempt=0 + # each attempts would take about 15 seconds + local maxAttempts=20 + until check_salt_minion_status; do + attempt=$((attempt+1)) + if [[ $attempt -eq $maxAttempts ]]; then + fail_setup + fi + sleep 10 + done +} + verify_setup() { info "Verifying setup" set -o pipefail