diff --git a/salt/common/init.sls b/salt/common/init.sls index 17cea3480..e511308a7 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -4,6 +4,11 @@ {% set role = grains.id.split('_') | last %} {% from 'elasticsearch/auth.map.jinja' import ELASTICAUTH with context %} +{% if grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-managersearch', 'so-import'] %} +include: + - manager.elasticsearch # needed for elastic_curl_config state +{% endif %} + # Remove variables.txt from /tmp - This is temp rmvariablesfile: file.absent: @@ -182,6 +187,7 @@ alwaysupdated: Etc/UTC: timezone.system +{% if salt['pillar.get']('elasticsearch:auth:enabled', False) %} elastic_curl_config: file.managed: - name: /opt/so/conf/elasticsearch/curl.config @@ -189,6 +195,11 @@ elastic_curl_config: - mode: 600 - show_changes: False - makedirs: True + {% if grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-managersearch', 'so-import'] %} + - require: + - file: elastic_curl_config_distributed + {% endif %} +{% endif %} # Sync some Utilities utilsyncscripts: diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 66c91aa7b..584e57926 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -360,6 +360,13 @@ run_check_net_err() { exit $exit_code fi } +set_cron_service_name() { + if [[ "$OS" == "centos" ]]; then + cron_service_name="crond" + else + cron_service_name="cron" + fi +} set_os() { if [ -f /etc/redhat-release ]; then diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 71891a57e..8e2a7d332 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -468,6 +468,51 @@ post_to_2.3.100() { echo "Post Processing for .100" } +stop_salt_master() { + # kill all salt jobs across the grid because the hang indefinitely if they are queued and salt-master restarts + set +e + echo "" + echo "Killing all Salt jobs across the grid." + salt \* saltutil.kill_all_jobs + echo "" + echo "Killing any queued Salt jobs on the manager." + pkill -9 -ef "/usr/bin/python3 /bin/salt" + set -e + + echo "" + echo "Storing salt-master pid." + MASTERPID=$(pgrep salt-master | head -1) + echo "Found salt-master PID $MASTERPID" + echo "" + echo "Stopping Salt Master service at $(date +"%T.%6N")" + systemctl stop salt-master + echo "" + timeout 30 tail --pid=$MASTERPID -f /dev/null || echo "salt-master still running at $(date +"%T.%6N") after waiting 30s. We cannot kill due to systemd restart option." +} + +stop_salt_minion() { + echo "Disabling highstate to prevent from running if salt-minion restarts." + salt-call state.disable highstate -l info --local + echo "" + + # kill all salt jobs before stopping salt-minion + set +e + echo "" + echo "Killing Salt jobs on this node." + salt-call saltutil.kill_all_jobs --local + set -e + + echo "Storing salt-minion pid." + MINIONPID=$(pgrep salt-minion | head -1) + echo "Found salt-minion PID $MINIONPID" + echo "Stopping Salt Minion service at $(date +"%T.%6N")." + systemctl stop salt-minion + + set +e + timeout 30 tail --pid=$MINIONPID -f /dev/null || echo "Killing salt-minion at $(date +"%T.%6N") after waiting 30s" && pkill -9 -ef /usr/bin/salt-minion + set -e +} + up_to_2.3.20(){ DOCKERSTUFFBIP=$(echo $DOCKERSTUFF | awk -F'.' '{print $1,$2,$3,1}' OFS='.')/24 # Remove PCAP from global @@ -784,11 +829,11 @@ upgrade_salt() { echo "Removing yum versionlock for Salt." echo "" yum versionlock delete "salt-*" - echo "Updating Salt packages and restarting services." + echo "Updating Salt packages." echo "" set +e run_check_net_err \ - "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -r -F -M -x python3 stable \"$NEWSALTVERSION\"" \ + "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -X -r -F -M -x python3 stable \"$NEWSALTVERSION\"" \ "Could not update salt, please check $SOUP_LOG for details." set -e echo "Applying yum versionlock for Salt." @@ -801,11 +846,11 @@ upgrade_salt() { apt-mark unhold "salt-common" apt-mark unhold "salt-master" apt-mark unhold "salt-minion" - echo "Updating Salt packages and restarting services." + echo "Updating Salt packages." echo "" set +e run_check_net_err \ - "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -F -M -x python3 stable \"$NEWSALTVERSION\"" \ + "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -X -F -M -x python3 stable \"$NEWSALTVERSION\"" \ "Could not update salt, please check $SOUP_LOG for details." set -e echo "Applying apt hold for Salt." @@ -913,7 +958,18 @@ fix_wazuh() { main() { trap 'check_err $?' EXIT + if [ -n "$BRANCH" ]; then + echo "SOUP will use the $BRANCH branch." + echo "" + fi + echo "### Preparing soup at $(date) ###" + echo "" + + echo "Checking to see if this is a manager." + echo "" + require_manager + check_pillar_items echo "Checking to see if this is an airgap install." @@ -923,9 +979,7 @@ main() { echo "Missing file argument (-f ) for unattended airgap upgrade." exit 0 fi - echo "Checking to see if this is a manager." - echo "" - require_manager + set_minionid echo "Found that Security Onion $INSTALLEDVERSION is currently installed." echo "" @@ -943,6 +997,7 @@ main() { verify_latest_update_script echo "" set_os + set_cron_service_name set_palette check_elastic_license echo "" @@ -976,6 +1031,10 @@ main() { echo "Performing upgrade from Security Onion $INSTALLEDVERSION to Security Onion $NEWVERSION." echo "" + echo "Stopping $cron_service_name service at $(date +"%T.%6N")." + echo "" + systemctl stop "$cron_service_name" + # update mine items prior to stopping salt-minion and salt-master update_salt_mine @@ -992,17 +1051,9 @@ main() { set -e fi - echo "" - echo "Stopping Salt Minion service." - systemctl stop salt-minion - echo "Killing any remaining Salt Minion processes." - set +e - pkill -9 -ef /usr/bin/salt-minion - set -e - echo "" - echo "Stopping Salt Master service." - systemctl stop salt-master - echo "" + stop_salt_minion + + stop_salt_master upgrade_to_2.3.50_repo @@ -1048,11 +1099,11 @@ main() { update_version echo "" - echo "Locking down Salt Master for upgrade" + echo "Locking down Salt Master for upgrade at $(date +"%T.%6N")." masterlock echo "" - echo "Starting Salt Master service." + echo "Starting Salt Master service at $(date +"%T.%6N")." systemctl start salt-master # Testing that salt-master is up by checking that is it connected to itself @@ -1066,6 +1117,13 @@ main() { salt-call state.apply salt.python3-influxdb -l info queue=True echo "" + # update the salt-minion configs here and start the minion + # since highstate are disabled above, minion start should not trigger a highstate + echo "" + echo "Ensuring salt-minion configs are up-to-date." + salt-call state.apply salt.minion -l info queue=True + echo "" + # Only regenerate osquery packages if Fleet is enabled FLEET_MANAGER=$(lookup_pillar fleet_manager) FLEET_NODE=$(lookup_pillar fleet_node) @@ -1076,20 +1134,22 @@ main() { echo "" fi + echo "Enabling highstate." + salt-call state.enable highstate -l info --local echo "" - echo "Running a highstate to complete the Security Onion upgrade on this manager. This could take several minutes." + + echo "" + echo "Running a highstate. This could take several minutes." set +e salt-call state.highstate -l info queue=True set -e - echo "" - echo "Stopping Salt Master to remove ACL" - systemctl stop salt-master + stop_salt_master masterunlock echo "" - echo "Starting Salt Master service." + echo "Starting Salt Master service at $(date +"%T.%6N") ." systemctl start salt-master set +e @@ -1097,7 +1157,7 @@ main() { salt-call state.show_top -l error queue=True || fail "salt-master could not be reached. Check $SOUP_LOG for details." set -e - echo "Running a highstate. This could take several minutes." + echo "Running a highstate to complete the Security Onion upgrade on this manager. This could take several minutes." salt-call state.highstate -l info queue=True postupgrade_changes [[ $is_airgap -eq 0 ]] && unmount_update @@ -1145,6 +1205,9 @@ main() { esac fi + echo "Starting $cron_service_name service at $(date +"%T.%6N")." + systemctl start "$cron_service_name" + if [[ $NUM_MINIONS -gt 1 ]]; then cat << EOF @@ -1223,12 +1286,6 @@ https://blog.securityonion.net EOF - if [ -n "$BRANCH" ]; then - cat << EOF -SOUP will use the $BRANCH branch. - -EOF - fi cat << EOF Press Enter to continue or Ctrl-C to cancel. EOF diff --git a/salt/cron/dead.sls b/salt/cron/dead.sls new file mode 100644 index 000000000..8ddb79cbf --- /dev/null +++ b/salt/cron/dead.sls @@ -0,0 +1,6 @@ +{% from "cron/map.jinja" import cronmap with context %} + +crond_service: + service.dead: + - name: {{ cronmap.service }} + - enable: True diff --git a/salt/cron/map.jinja b/salt/cron/map.jinja new file mode 100644 index 000000000..4f5b78823 --- /dev/null +++ b/salt/cron/map.jinja @@ -0,0 +1,8 @@ +{% set cronmap = salt['grains.filter_by']({ + 'Debian': { + 'service': 'cron', + }, + 'RedHat': { + 'service': 'crond', + }, +}) %} diff --git a/salt/cron/running.sls b/salt/cron/running.sls new file mode 100644 index 000000000..bcd5e7ef9 --- /dev/null +++ b/salt/cron/running.sls @@ -0,0 +1,7 @@ +{% from "cron/map.jinja" import cronmap with context %} + +crond_service: + service.running: + - name: {{ cronmap.service }} + - enable: True + - unless: pgrep soup diff --git a/salt/elasticsearch/files/curl.config.template b/salt/elasticsearch/files/curl.config.template index 14f5a2a1d..9c057cabf 100644 --- a/salt/elasticsearch/files/curl.config.template +++ b/salt/elasticsearch/files/curl.config.template @@ -1 +1 @@ -user = "{{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user') }}:{{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass') }}" \ No newline at end of file +user = "{{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', 'NO_USER_SET') }}:{{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', 'NO_PW_SET') }}" diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index e24eab25e..78a586428 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -53,7 +53,7 @@ vm.max_map_count: cascriptsync: file.managed: - name: /usr/sbin/so-catrust - - source: salt://elasticsearch/files/scripts/so-catrust + - source: salt://elasticsearch/tools/sbin/so-catrust - user: 939 - group: 939 - mode: 750 @@ -63,9 +63,37 @@ cascriptsync: cascriptfun: cmd.run: - name: /usr/sbin/so-catrust - + - require: + - file: cascriptsync {% endif %} +# Sync some es scripts +es_sync_scripts: + file.recurse: + - name: /usr/sbin + - user: root + - group: root + - file_mode: 755 + - template: jinja + - source: salt://elasticsearch/tools/sbin + - defaults: + ELASTICCURL: 'curl' + - context: + ELASTICCURL: {{ ELASTICAUTH.elasticcurl }} + - exclude_pat: + - so-elasticsearch-pipelines # exclude this because we need to watch it for changes, we sync it in another state + +so-elasticsearch-pipelines-script: + file.managed: + - name: /usr/sbin/so-elasticsearch-pipelines + - source: salt://elasticsearch/tools/sbin/so-elasticsearch-pipelines + - user: 930 + - group: 939 + - mode: 754 + - template: jinja + - defaults: + ELASTICCURL: {{ ELASTICAUTH.elasticcurl }} + # Move our new CA over so Elastic and Logstash can use SSL with the internal CA catrustdir: file.directory: @@ -297,7 +325,7 @@ so-elasticsearch: - file: esyml - file: esingestconf - file: esingestdynamicconf - - file: so-elasticsearch-pipelines-file + - file: so-elasticsearch-pipelines-script - require: - file: esyml - file: eslog4jfile @@ -322,27 +350,17 @@ append_so-elasticsearch_so-status.conf: - name: /opt/so/conf/so-status/so-status.conf - text: so-elasticsearch -so-elasticsearch-pipelines-file: - file.managed: - - name: /opt/so/conf/elasticsearch/so-elasticsearch-pipelines - - source: salt://elasticsearch/files/so-elasticsearch-pipelines - - user: 930 - - group: 939 - - mode: 754 - - template: jinja - - defaults: - ELASTICCURL: {{ ELASTICAUTH.elasticcurl }} - so-elasticsearch-pipelines: cmd.run: - - name: /opt/so/conf/elasticsearch/so-elasticsearch-pipelines {{ grains.host }} + - name: /usr/sbin/so-elasticsearch-pipelines {{ grains.host }} - onchanges: - file: esingestconf - file: esingestdynamicconf - file: esyml - - file: so-elasticsearch-pipelines-file + - file: so-elasticsearch-pipelines-script - require: - docker_container: so-elasticsearch + - file: so-elasticsearch-pipelines-script {% if TEMPLATES %} so-elasticsearch-templates: @@ -352,6 +370,7 @@ so-elasticsearch-templates: - template: jinja - require: - docker_container: so-elasticsearch + - file: es_sync_scripts {% endif %} so-elasticsearch-roles-load: @@ -361,6 +380,7 @@ so-elasticsearch-roles-load: - template: jinja - require: - docker_container: so-elasticsearch + - file: es_sync_scripts {% endif %} {# if grains['role'] != 'so-helix' #} diff --git a/salt/elasticsearch/files/scripts/so-catrust b/salt/elasticsearch/tools/sbin/so-catrust similarity index 100% rename from salt/elasticsearch/files/scripts/so-catrust rename to salt/elasticsearch/tools/sbin/so-catrust diff --git a/salt/elasticsearch/files/so-elasticsearch-pipelines b/salt/elasticsearch/tools/sbin/so-elasticsearch-pipelines similarity index 100% rename from salt/elasticsearch/files/so-elasticsearch-pipelines rename to salt/elasticsearch/tools/sbin/so-elasticsearch-pipelines diff --git a/salt/common/tools/sbin/so-elasticsearch-roles-load b/salt/elasticsearch/tools/sbin/so-elasticsearch-roles-load similarity index 100% rename from salt/common/tools/sbin/so-elasticsearch-roles-load rename to salt/elasticsearch/tools/sbin/so-elasticsearch-roles-load diff --git a/salt/common/tools/sbin/so-elasticsearch-templates-load b/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load similarity index 100% rename from salt/common/tools/sbin/so-elasticsearch-templates-load rename to salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load diff --git a/salt/manager/elasticsearch.sls b/salt/manager/elasticsearch.sls new file mode 100644 index 000000000..24c509fb4 --- /dev/null +++ b/salt/manager/elasticsearch.sls @@ -0,0 +1,9 @@ +{% if salt['pillar.get']('elasticsearch:auth:enabled', False) %} +elastic_curl_config_distributed: + file.managed: + - name: /opt/so/saltstack/local/salt/elasticsearch/curl.config + - source: salt://elasticsearch/files/curl.config.template + - template: jinja + - mode: 600 + - show_changes: False +{% endif %} diff --git a/salt/manager/init.sls b/salt/manager/init.sls index 990eda3d3..c913383b0 100644 --- a/salt/manager/init.sls +++ b/salt/manager/init.sls @@ -21,10 +21,10 @@ {% set STRELKA_RULES = salt['pillar.get']('strelka:rules', '1') %} include: - - elasticsearch.auth - - kibana.secrets - salt.minion - - kratos + - kibana.secrets + - manager.sync_es_users + - manager.elasticsearch socore_own_saltstack: file.directory: @@ -110,29 +110,6 @@ strelka_yara_update: - hour: '7' - minute: '1' -elastic_curl_config_distributed: - file.managed: - - name: /opt/so/saltstack/local/salt/elasticsearch/curl.config - - source: salt://elasticsearch/files/curl.config.template - - template: jinja - - mode: 600 - - show_changes: False - -# Must run before elasticsearch docker container is started! -syncesusers: - cmd.run: - - name: so-user sync - - env: - - SKIP_STATE_APPLY: 'true' - - creates: - - /opt/so/saltstack/local/salt/elasticsearch/files/users - - /opt/so/saltstack/local/salt/elasticsearch/files/users_roles - - /opt/so/conf/soc/soc_users_roles - - show_changes: False - - require: - - docker_container: so-kratos - - http: wait_for_kratos - {% else %} {{sls}}_state_not_allowed: diff --git a/salt/manager/sync_es_users.sls b/salt/manager/sync_es_users.sls new file mode 100644 index 000000000..4546fc52f --- /dev/null +++ b/salt/manager/sync_es_users.sls @@ -0,0 +1,31 @@ +include: + - elasticsearch.auth + - kratos + +so-user.lock: + file.missing: + - name: /var/tmp/so-user.lock + +# Must run before elasticsearch docker container is started! +sync_es_users: + cmd.run: + - name: so-user sync + - env: + - SKIP_STATE_APPLY: 'true' + - creates: + - /opt/so/saltstack/local/salt/elasticsearch/files/users + - /opt/so/saltstack/local/salt/elasticsearch/files/users_roles + - /opt/so/conf/soc/soc_users_roles + - show_changes: False + - require: + - docker_container: so-kratos + - http: wait_for_kratos + - file: so-user.lock # require so-user.lock file to be missing + +# we dont want this added too early in setup, so we add the onlyif to verify 'startup_states: highstate' +# is in the minion config. That line is added before the final highstate during setup +sosyncusers: + cron.present: + - user: root + - name: 'PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin /usr/sbin/so-user sync &>> /opt/so/log/soc/sync.log' + - onlyif: "grep 'startup_states: highstate' /etc/salt/minion" diff --git a/salt/salt/enable_highstate.sls b/salt/salt/enable_highstate.sls new file mode 100644 index 000000000..72e5c1410 --- /dev/null +++ b/salt/salt/enable_highstate.sls @@ -0,0 +1,7 @@ +enable_highstate: + module.run: + - state.enable: + - states: + - highstate + - unless: pgrep soup + \ No newline at end of file diff --git a/salt/salt/minion.sls b/salt/salt/minion.sls index 38f8889c3..a9320defb 100644 --- a/salt/salt/minion.sls +++ b/salt/salt/minion.sls @@ -62,8 +62,6 @@ set_log_levels: - text: - "log_level: error" - "log_level_logfile: error" - - listen_in: - - service: salt_minion_service salt_minion_service_unit_file: file.managed: @@ -74,8 +72,6 @@ salt_minion_service_unit_file: service_start_delay: {{ service_start_delay }} - onchanges_in: - module: systemd_reload - - listen_in: - - service: salt_minion_service {% endif %} @@ -91,8 +87,13 @@ salt_minion_service: - name: salt-minion - enable: True - onlyif: test "{{INSTALLEDSALTVERSION}}" == "{{SALTVERSION}}" - - watch: + - listen: - file: mine_functions +{% if INSTALLEDSALTVERSION|string == SALTVERSION|string %} + - file: set_log_levels + - file: salt_minion_service_unit_file +{% endif %} + patch_pkg: pkg.installed: diff --git a/salt/soc/init.sls b/salt/soc/init.sls index 856f929bd..bfb6ea4d9 100644 --- a/salt/soc/init.sls +++ b/salt/soc/init.sls @@ -5,6 +5,9 @@ {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% set MANAGER = salt['grains.get']('master') %} +include: + - manager.sync_es_users + socdir: file.directory: - name: /opt/so/conf/soc @@ -84,14 +87,8 @@ soccustomroles: socusersroles: file.exists: - name: /opt/so/conf/soc/soc_users_roles - -# we dont want this added too early in setup, so we add the onlyif to verify 'startup_states: highstate' -# is in the minion config. That line is added before the final highstate during setup -sosyncusers: - cron.present: - - user: root - - name: 'PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin /usr/sbin/so-user sync &>> /opt/so/log/soc/sync.log' - - onlyif: "grep 'startup_states: highstate' /etc/salt/minion" + - require: + - sls: manager.sync_es_users so-soc: docker_container.running: diff --git a/salt/top.sls b/salt/top.sls index 4fd8c1fd3..513439255 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -19,6 +19,10 @@ base: + '*': + - salt.enable_highstate + - cron.running + 'not G@saltversion:{{saltversion}}': - match: compound - salt.minion-state-apply-test diff --git a/setup/so-functions b/setup/so-functions index ff7904e61..6bc7bed70 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2144,7 +2144,7 @@ restore_file() { if [ -f "$src" ]; then [ ! -d "$dst" ] && mkdir -v -p "$dst" echo "Restoring $src to $dst." >> "$setup_log" 2>&1 - cp -v "$src" "$dst" + cp -v "$src" "$dst" >> "$setup_log" 2>&1 fi }