diff --git a/salt/backup/tools/sbin/so-config-backup.jinja b/salt/backup/tools/sbin/so-config-backup.jinja index 7f65bbba3..8d214e665 100755 --- a/salt/backup/tools/sbin/so-config-backup.jinja +++ b/salt/backup/tools/sbin/so-config-backup.jinja @@ -25,9 +25,11 @@ if [ ! -f $BACKUPFILE ]; then # Create empty backup file tar -cf $BACKUPFILE -T /dev/null - # Loop through all paths defined in global.sls, and append them to backup file + # Loop through all paths defined in global.sls, and append them to backup file if they exist {%- for LOCATION in BACKUPLOCATIONS %} - tar -rf $BACKUPFILE "${EXCLUSIONS[@]}" {{ LOCATION }} + if [[ -d {{ LOCATION }} || -f {{ LOCATION }} ]]; then + tar -rf $BACKUPFILE "${EXCLUSIONS[@]}" {{ LOCATION }} + fi {%- endfor %} fi diff --git a/salt/elasticfleet/manager.sls b/salt/elasticfleet/manager.sls index 1728f2010..6cb672bef 100644 --- a/salt/elasticfleet/manager.sls +++ b/salt/elasticfleet/manager.sls @@ -11,14 +11,15 @@ include: - elasticfleet.config # If enabled, automatically update Fleet Logstash Outputs -{% if ELASTICFLEETMERGED.config.server.enable_auto_configuration and grains.role not in ['so-import', 'so-eval'] %} +{% if ELASTICFLEETMERGED.config.server.enable_auto_configuration %} +{% if grains.role not in ['so-import', 'so-eval']%} so-elastic-fleet-auto-configure-logstash-outputs: cmd.run: - name: /usr/sbin/so-elastic-fleet-outputs-update - retry: attempts: 4 interval: 30 -{% endif %} +{% endif %} # If enabled, automatically update Fleet Server URLs & ES Connection so-elastic-fleet-auto-configure-server-urls: @@ -27,6 +28,7 @@ so-elastic-fleet-auto-configure-server-urls: - retry: attempts: 4 interval: 30 +{% endif %} # Automatically update Fleet Server Elasticsearch URLs & Agent Artifact URLs so-elastic-fleet-auto-configure-elasticsearch-urls: diff --git a/salt/elasticsearch/cluster.sls b/salt/elasticsearch/cluster.sls index e25aed36a..d20ee45ca 100644 --- a/salt/elasticsearch/cluster.sls +++ b/salt/elasticsearch/cluster.sls @@ -9,9 +9,12 @@ {% from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %} {% from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS, SO_MANAGED_INDICES %} {% if GLOBALS.role != 'so-heavynode' %} -{% from 'elasticsearch/template.map.jinja' import ALL_ADDON_SETTINGS %} +{% from 'elasticsearch/template.map.jinja' import ALL_ADDON_SETTINGS, ADDON_INDICES %} {% endif %} +include: + - elasticsearch.enabled + escomponenttemplates: file.recurse: - name: /opt/so/conf/elasticsearch/templates/component @@ -35,6 +38,20 @@ so_index_template_dir: {%- endfor %} {%- endif %} +{% if GLOBALS.role != "so-heavynode" %} +# Clean up legacy and non-SO managed templates from the elasticsearch/templates/addon-index/ directory +addon_index_template_dir: + file.directory: + - name: /opt/so/conf/elasticsearch/templates/addon-index + - clean: True + {%- if ADDON_INDICES %} + - require: + {%- for index in ADDON_INDICES %} + - file: addon_index_template_{{index}} + {%- endfor %} + {%- endif %} +{% endif %} + # Auto-generate index templates for SO managed indices (directly defined in elasticsearch/defaults.yaml) # These index templates are for the core SO datasets and are always required {% for index, settings in ES_INDEX_SETTINGS.items() %} diff --git a/salt/elasticsearch/template.map.jinja b/salt/elasticsearch/template.map.jinja index e66057775..ed1b49abe 100644 --- a/salt/elasticsearch/template.map.jinja +++ b/salt/elasticsearch/template.map.jinja @@ -61,15 +61,25 @@ {% if ALL_ADDON_SETTINGS_ORIG.keys() | length > 0 %} {% for index in ALL_ADDON_SETTINGS_ORIG.keys() %} {% do ALL_ADDON_SETTINGS_GLOBAL_OVERRIDES.update({index: salt['defaults.merge'](ALL_ADDON_SETTINGS_ORIG[index], PILLAR_GLOBAL_OVERRIDES, in_place=False)}) %} +{# Explicitly excluding addon indices from ES_INDEX_SETTINGS_ORIG + When manager.soc_managed_annotations runs, new entries are added to the salt/elasticsearch/defaults.yaml file to support 'revert to default' functionality. + Subsequent map renders will then incorrectly include 'integration X' in 'ES_INDEX_SETTINGS_ORIG' due to being in the defaults.yaml file. #} +{% if index in ES_INDEX_SETTINGS_ORIG.keys() %} +{% do ES_INDEX_SETTINGS_ORIG.pop(index) %} +{% endif %} {% endfor %} {% endif %} {% set ES_INDEX_SETTINGS = {} %} -{% macro create_final_index_template(DEFINED_SETTINGS, GLOBAL_OVERRIDES, FINAL_INDEX_SETTINGS) %} +{% macro create_final_index_template(DEFINED_SETTINGS, GLOBAL_OVERRIDES, FINAL_INDEX_SETTINGS, EXCLUDE_INDICES=[]) %} {% do GLOBAL_OVERRIDES.update(salt['defaults.merge'](GLOBAL_OVERRIDES, ES_INDEX_PILLAR, in_place=False)) %} {% for index, settings in GLOBAL_OVERRIDES.items() %} +{% if index in EXCLUDE_INDICES %} +{% continue %} +{% endif %} + {# prevent this action from being performed on custom defined indices. #} {# the custom defined index is not present in either of the dictionaries and fails to reder. #} {% if index in DEFINED_SETTINGS and index in GLOBAL_OVERRIDES %} @@ -150,10 +160,19 @@ {% endfor %} {% endmacro %} -{{ create_final_index_template(ES_INDEX_SETTINGS_ORIG, ES_INDEX_SETTINGS_GLOBAL_OVERRIDES, ES_INDEX_SETTINGS) }} -{{ create_final_index_template(ALL_ADDON_SETTINGS_ORIG, ALL_ADDON_SETTINGS_GLOBAL_OVERRIDES, ALL_ADDON_SETTINGS) }} +{# Exclude addon integrations from final ES_INDEX_SETTINGS #} +{{ create_final_index_template(ES_INDEX_SETTINGS_ORIG, ES_INDEX_SETTINGS_GLOBAL_OVERRIDES, ES_INDEX_SETTINGS, ALL_ADDON_SETTINGS_ORIG.keys() | list ) }} + +{# Exclude SO managed indices, otherwise ALL_ADDON_SETTINGS will include pillar values + of core integrations without merging defaults, resulting in an overlapping, but bad index template being generated. #} +{{ create_final_index_template(ALL_ADDON_SETTINGS_ORIG, ALL_ADDON_SETTINGS_GLOBAL_OVERRIDES, ALL_ADDON_SETTINGS, ES_INDEX_SETTINGS_ORIG.keys() | list ) }} {% set SO_MANAGED_INDICES = [] %} {% for index, settings in ES_INDEX_SETTINGS.items() %} {% do SO_MANAGED_INDICES.append(index) %} -{% endfor %} \ No newline at end of file +{% endfor %} + +{% set ADDON_INDICES = [] %} +{% for index, settings in ALL_ADDON_SETTINGS.items() %} +{% do ADDON_INDICES.append(index) %} +{% endfor %} diff --git a/salt/manager/sync_es_users.sls b/salt/manager/sync_es_users.sls index 29b090e18..8fc9c6bb4 100644 --- a/salt/manager/sync_es_users.sls +++ b/salt/manager/sync_es_users.sls @@ -31,11 +31,13 @@ sync_es_users: - http: wait_for_kratos - file: so-user.lock # require so-user.lock file to be missing -# we dont want this added too early in setup, so we add the onlyif to verify 'startup_states: highstate' -# is in the minion config. That line is added before the final highstate during setup +# we dont want this added too early in setup, so the onlyif gates on the +# /opt/so/state/setup-complete marker. The marker is written by +# mark_setup_complete in setup/so-functions just before the final setup +# highstate (and by an upgrade-path state for systems set up under the old gate). so-user_sync: cron.present: - user: root - name: 'PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin /usr/sbin/so-user sync &>> /opt/so/log/soc/sync.log' - identifier: so-user_sync - - onlyif: "grep -x 'startup_states: highstate' /etc/salt/minion" + - onlyif: "test -e /opt/so/state/setup-complete" diff --git a/salt/manager/tools/sbin/so-boot-mine-update b/salt/manager/tools/sbin/so-boot-mine-update new file mode 100755 index 000000000..79cd67844 --- /dev/null +++ b/salt/manager/tools/sbin/so-boot-mine-update @@ -0,0 +1,117 @@ +#!/bin/bash +# +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +# Runs once per boot on managers (via so-boot-mine-update.service), before +# so-boot-highstate.service. Waits for the responsive minion set to settle, pushes +# mine.update, waits until every up minion has actually reported to the mine, then +# warms the master's per-minion pillar cache so the mine-backed node pillars (node +# IPs, ES/Redis/Logstash/hypervisor discovery -- some glob- and some pillar/grain- +# targeted) are complete before the boot highstate renders them. Otherwise a node +# that is up but not yet fully reported gets dropped from those pillars and torn +# out of the configs they build (e.g. so-elasticsearch ExtraHosts -> container recreate). + +MAX_WAIT=${MINE_UPDATE_MAX_WAIT:-180} # hard backstop only +INTERVAL=10 +STABLE_CHECKS=3 # up-count must hold steady this many polls +elapsed=0 +prev=-1 +stable=0 +up=0 + +# Wait for the *reachable* minion set to settle rather than for every accepted +# key to report up: an operator may accept a minion's key and then intentionally +# power off that host, so requiring up >= accepted would never be satisfied and +# we'd always burn the full MAX_WAIT. Once the responsive count stops growing we +# stop waiting and run mine.update against whoever is up. +while [ "$elapsed" -lt "$MAX_WAIT" ]; do + up=$(/usr/bin/salt-run manage.up --out=json 2>/dev/null \ + | python3 -c 'import sys,json; print(len(json.load(sys.stdin)))' 2>/dev/null) + up=${up:-0} + if [ "$up" -gt 0 ] && [ "$up" -eq "$prev" ]; then + stable=$((stable + 1)) + [ "$stable" -ge "$STABLE_CHECKS" ] && break + else + stable=0 + fi + prev=$up + sleep "$INTERVAL" + elapsed=$((elapsed + INTERVAL)) +done + +echo "so-boot-mine-update: ${up} minions up (settled after ${elapsed}s); running mine.update" +/usr/bin/salt '*' mine.update --out=txt + +# A node that is up but has not yet re-reported network.ip_addrs to the mine is +# silently dropped from mine-backed pillars (elasticsearch:nodes, node_data, ...) +# when highstate recompiles them -- which e.g. removes it from so-elasticsearch +# ExtraHosts and forces a container recreate. After the broad mine.update above, +# wait until every up minion actually has network.ip_addrs in the mine, re-pushing +# mine.update to stragglers, before releasing the boot highstate. Bounded by the +# same MAX_WAIT backstop so a slow/down node never blocks boot indefinitely. +missing="" +while [ "$elapsed" -lt "$MAX_WAIT" ]; do + up_json=$(/usr/bin/salt-run manage.up --out=json 2>/dev/null) + mine_json=$(/usr/bin/salt-run mine.get '*' network.ip_addrs tgt_type=glob --out=json 2>/dev/null) + missing=$(printf '%s' "$up_json" | python3 -c ' +import sys, json +up = set(json.load(sys.stdin) or []) +mine = {k for k, v in (json.loads(sys.argv[1]) or {}).items() if v} +print("\n".join(sorted(up - mine))) +' "$mine_json" 2>/dev/null) + if [ -z "$missing" ]; then + echo "so-boot-mine-update: mine complete for all up minions after ${elapsed}s" + break + fi + echo "so-boot-mine-update: mine missing up minion(s): $(echo $missing); re-running mine.update" + for m in $missing; do /usr/bin/salt "$m" mine.update --out=txt; done + sleep "$INTERVAL" + elapsed=$((elapsed + INTERVAL)) +done +[ -n "$missing" ] && echo "so-boot-mine-update: WARNING ${MAX_WAIT}s backstop hit; up minion(s) still absent from mine: $(echo $missing); highstate may drop them from configs" + +# The pillar/compound-targeted node pillars (elasticsearch:nodes, redis:nodes, +# logstash:nodes, hypervisor:nodes) resolve their target against the master's +# per-minion data cache (grains+pillar in .../minions//data.p), populated only +# when a minion's pillar is (re)compiled -- separately from the mine. A freshly +# booted node can be in the mine (glob/node_data sees it) yet absent from that +# cache, so it is dropped from those pillars and from the configs they build (e.g. +# so-elasticsearch ExtraHosts). Force a synchronous pillar refresh so the master +# caches every up node's pillar; refresh_pillar wait=True returns only once the +# pillar is recompiled (and thus cached for matching). Retry stragglers <= MAX_WAIT. +echo "so-boot-mine-update: warming master pillar cache for pillar/grain-targeted node pillars" +/usr/bin/salt '*' saltutil.refresh_pillar wait=True --out=txt +missing="" +while [ "$elapsed" -lt "$MAX_WAIT" ]; do + up_json=$(/usr/bin/salt-run manage.up --out=json 2>/dev/null) + cached_json=$(/usr/bin/salt-run cache.pillar tgt='*' --out=json 2>/dev/null) + missing=$(printf '%s' "$up_json" | python3 -c ' +import sys, json +up = set(json.load(sys.stdin) or []) +cached = {k for k, v in (json.loads(sys.argv[1]) or {}).items() if v} +print("\n".join(sorted(up - cached))) +' "$cached_json" 2>/dev/null) + if [ -z "$missing" ]; then + echo "so-boot-mine-update: pillar cache warm for all up minions after ${elapsed}s" + break + fi + echo "so-boot-mine-update: pillar not yet cached for: $(echo $missing); refreshing" + for m in $missing; do /usr/bin/salt "$m" saltutil.refresh_pillar wait=True --out=txt; done + sleep "$INTERVAL" + elapsed=$((elapsed + INTERVAL)) +done +[ -n "$missing" ] && echo "so-boot-mine-update: WARNING ${MAX_WAIT}s backstop hit; pillar not cached for: $(echo $missing); pillar-targeted pillars may drop them" + +# Log what the mine-backed pillars render so the boot-time state is inspectable. +/usr/bin/salt-call saltutil.refresh_pillar >/dev/null 2>&1 +sleep 2 +for key in node_data elasticsearch:nodes; do + rendered=$(/usr/bin/salt-call --out=json pillar.get "$key" 2>/dev/null \ + | python3 -c 'import sys,json; print(json.dumps(json.load(sys.stdin).get("local"), indent=2, sort_keys=True))' 2>/dev/null) + echo "so-boot-mine-update: ${key} rendered as:" + echo "${rendered:-null}" +done +exit 0 diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 135c51276..d50187c9c 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -188,13 +188,6 @@ airgap_update_dockers() { fi } -backup_old_states_pillars() { - - tar czf /nsm/backup/$(echo $INSTALLEDVERSION)_$(date +%Y%m%d-%H%M%S)_soup_default_states_pillars.tar.gz /opt/so/saltstack/default/ - tar czf /nsm/backup/$(echo $INSTALLEDVERSION)_$(date +%Y%m%d-%H%M%S)_soup_local_states_pillars.tar.gz /opt/so/saltstack/local/ - -} - update_registry() { docker stop so-dockerregistry docker rm so-dockerregistry @@ -769,12 +762,18 @@ bootstrap_so_soc_database() { } up_to_3.2.0() { + fix_logstash_0013_lumberjack_pipeline_name + INSTALLEDVERSION=3.2.0 } post_to_3.2.0() { bootstrap_so_soc_database + # Including agent regen script here since it was missed in post_to_3.1.0 + echo "Regenerating Elastic Agent Installers" + /sbin/so-elastic-agent-gen-installers + POSTVERSION=3.2.0 } @@ -1566,13 +1565,7 @@ EOF # Keeping this block in case we need to do a hotfix that requires salt update apply_hotfix() { - if [[ "$INSTALLEDVERSION" == "3.1.0" ]] ; then - # Do not remove this fix_logstash_0013_lumberjack_pipeline_name in future hotfixes without first validating older - # installs referencing "so/0013_input_lumberjack_fleet.conf" via pillar are upgradable - fix_logstash_0013_lumberjack_pipeline_name - else - echo "No actions required. ($INSTALLEDVERSION/$HOTFIXVERSION)" - fi + echo "No actions required. ($INSTALLEDVERSION/$HOTFIXVERSION)" } failed_soup_restore_items() { @@ -1644,13 +1637,13 @@ main() { echo "Verifying we have the latest soup script." verify_latest_update_script - echo "Verifying Elasticsearch version compatibility across the grid before upgrading." - verify_es_version_compatibility - echo "Let's see if we need to update Security Onion." upgrade_check upgrade_space + echo "Verifying Elasticsearch version compatibility across the grid before upgrading." + verify_es_version_compatibility + echo "Checking for Salt Master and Minion updates." upgrade_check_salt set -e @@ -1670,7 +1663,8 @@ main() { echo "Applying $HOTFIXVERSION hotfix" # since we don't run the backup.config_backup state on import we wont snapshot previous version states and pillars if [[ ! "$MINION_ROLE" == "import" ]]; then - backup_old_states_pillars + echo "Running so-config-backup script." + /sbin/so-config-backup fi copy_new_files create_local_directories "/opt/so/saltstack/default" @@ -1726,8 +1720,8 @@ main() { # since we don't run the backup.config_backup state on import we wont snapshot previous version states and pillars if [[ ! "$MINION_ROLE" == "import" ]]; then echo "" - echo "Creating snapshots of default and local Salt states and pillars and saving to /nsm/backup/" - backup_old_states_pillars + echo "Running so-config-backup script." + /sbin/so-config-backup fi echo "" diff --git a/salt/postgres/files/init-db.sh b/salt/postgres/files/init-db.sh index 03e6d08dd..d12bc4c9b 100644 --- a/salt/postgres/files/init-db.sh +++ b/salt/postgres/files/init-db.sh @@ -17,6 +17,7 @@ psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-E END IF; END \$\$; + GRANT ALL ON SCHEMA public TO "$SO_POSTGRES_USER"; GRANT ALL PRIVILEGES ON DATABASE "$POSTGRES_DB" TO "$SO_POSTGRES_USER"; -- Lock the SOC database down at the connect layer; PUBLIC gets CONNECT -- by default, which would let per-minion telegraf roles open sessions @@ -31,9 +32,4 @@ EOSQL # only ensures the shared database exists on first initialization. if ! psql -U "$POSTGRES_USER" -tAc "SELECT 1 FROM pg_database WHERE datname='so_telegraf'" | grep -q 1; then psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -c "CREATE DATABASE so_telegraf" -fi - -# Bootstrap the SOC database. -if ! psql -U "$POSTGRES_USER" -tAc "SELECT 1 FROM pg_database WHERE datname='so_soc'" | grep -q 1; then - psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -c "CREATE DATABASE so_soc" -fi +fi \ No newline at end of file diff --git a/salt/salt/master.sls b/salt/salt/master.sls index 3833232fd..273e97e84 100644 --- a/salt/salt/master.sls +++ b/salt/salt/master.sls @@ -16,6 +16,7 @@ include: - salt.minion - salt.master.pyinotify + - salt.master.boot_mine_update {% if 'vrt' in salt['pillar.get']('features', []) %} - salt.cloud - salt.cloud.reactor_config_hypervisor diff --git a/salt/salt/master/boot_mine_update.sls b/salt/salt/master/boot_mine_update.sls new file mode 100644 index 000000000..9f96c0ddf --- /dev/null +++ b/salt/salt/master/boot_mine_update.sls @@ -0,0 +1,29 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +# Manages /etc/systemd/system/so-boot-mine-update.service, a manager-only +# Type=oneshot unit that pushes `salt '*' mine.update` once per boot, ordered +# before so-boot-highstate.service so mine-backed pillars (node IPs, ES/Redis/ +# Logstash discovery) are fresh before the boot highstate renders them. + +include: + - systemd.reload + +so_boot_mine_update_unit_file: + file.managed: + - name: /etc/systemd/system/so-boot-mine-update.service + - source: salt://salt/service/so-boot-mine-update.service + - onchanges_in: + - module: systemd_reload + +# Only enable once setup is complete. Until then the gate file is missing and +# the unit's own ConditionPathExists would no-op it anyway. +so_boot_mine_update_service: + service.enabled: + - name: so-boot-mine-update.service + - onlyif: test -e /opt/so/state/setup-complete + - require: + - file: so_boot_mine_update_unit_file + - module: systemd_reload diff --git a/salt/salt/minion/boot_highstate.sls b/salt/salt/minion/boot_highstate.sls new file mode 100644 index 000000000..eb2596dad --- /dev/null +++ b/salt/salt/minion/boot_highstate.sls @@ -0,0 +1,31 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +# Manages /etc/systemd/system/so-boot-highstate.service, a Type=oneshot +# RemainAfterExit=yes unit that runs `salt-call state.highstate` exactly once +# per system boot. Replaces the legacy `startup_states: highstate` minion +# config, which fired on every salt-minion service restart (causing a redundant +# highstate whenever a highstate itself restarted salt-minion). + +include: + - systemd.reload + +so_boot_highstate_unit_file: + file.managed: + - name: /etc/systemd/system/so-boot-highstate.service + - source: salt://salt/service/so-boot-highstate.service + - onchanges_in: + - module: systemd_reload + +# Only enable once setup is complete. Until then the gate file is missing and +# the unit's own ConditionPathExists would no-op it anyway -- this just keeps +# `systemctl is-enabled` honest for the sync_es_users gate. +so_boot_highstate_service: + service.enabled: + - name: so-boot-highstate.service + - onlyif: test -e /opt/so/state/setup-complete + - require: + - file: so_boot_highstate_unit_file + - module: systemd_reload diff --git a/salt/salt/minion/init.sls b/salt/salt/minion/init.sls index 01c24e698..a251aa633 100644 --- a/salt/salt/minion/init.sls +++ b/salt/salt/minion/init.sls @@ -17,6 +17,7 @@ include: - repo.client - salt.mine_functions - salt.minion.service_file + - salt.minion.boot_highstate {% if GLOBALS.is_manager %} - ca.signing_policy {% endif %} @@ -80,11 +81,33 @@ set_log_levels: - "log_level: info" - "log_level_logfile: info" -enable_startup_states: - file.uncomment: +# startup_states: highstate caused a full highstate to run on every +# salt-minion service start, including the restart triggered when a highstate +# itself modified the minion config (beacons, mine, unit file). Replaced by +# so-boot-highstate.service (managed in salt.minion.boot_highstate), which +# runs once per system boot only. Strip the line from /etc/salt/minion on +# upgrade; both the commented and uncommented forms historically existed. +remove_startup_states: + file.line: - name: /etc/salt/minion - - regex: '^startup_states: highstate$' - - unless: pgrep so-setup + - match: 'startup_states: highstate' + - mode: delete + +# Upgrade-path bridge: systems that already passed setup under the old gate +# (`grep -x 'startup_states: highstate' /etc/salt/minion`) get a /opt/so/state/setup-complete +# marker so so-boot-highstate.service can be enabled and the so-user_sync cron +# in sync_es_users.sls keeps installing. Setup-in-progress systems instead get +# the marker from `mark_setup_complete` in setup/so-functions at the right +# moment. `replace: false` means we never overwrite a marker once written. +mark_setup_complete_for_upgrades: + file.managed: + - name: /opt/so/state/setup-complete + - replace: false + - makedirs: True + - onlyif: "grep -qx 'startup_states: highstate' /etc/salt/minion" + - require_in: + - file: remove_startup_states + - service: so_boot_highstate_service {% endif %} diff --git a/salt/salt/service/so-boot-highstate.service b/salt/salt/service/so-boot-highstate.service new file mode 100644 index 000000000..cc8c6a1c6 --- /dev/null +++ b/salt/salt/service/so-boot-highstate.service @@ -0,0 +1,14 @@ +[Unit] +Description=Security Onion boot-time highstate (runs once per boot) +After=salt-minion.service network-online.target docker.service +Wants=network-online.target docker.service +Requires=salt-minion.service +ConditionPathExists=/opt/so/state/setup-complete + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/usr/bin/salt-call state.highstate -l info queue=True + +[Install] +WantedBy=multi-user.target diff --git a/salt/salt/service/so-boot-mine-update.service b/salt/salt/service/so-boot-mine-update.service new file mode 100644 index 000000000..c5c6cdf7b --- /dev/null +++ b/salt/salt/service/so-boot-mine-update.service @@ -0,0 +1,15 @@ +[Unit] +Description=Security Onion boot-time grid mine.update (managers, runs once per boot before highstate) +After=salt-master.service salt-minion.service network-online.target +Wants=network-online.target +Requires=salt-master.service salt-minion.service +Before=so-boot-highstate.service +ConditionPathExists=/opt/so/state/setup-complete + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/usr/sbin/so-boot-mine-update + +[Install] +WantedBy=multi-user.target diff --git a/salt/setup/virt/setSalt.sls b/salt/setup/virt/setSalt.sls index 69c8795de..59ab9e1e3 100644 --- a/salt/setup/virt/setSalt.sls +++ b/salt/setup/virt/setSalt.sls @@ -8,11 +8,6 @@ set_role_grain: - name: role - value: so-{{ grains.id.split("_") | last }} -set_highstate: - file.append: - - name: /etc/salt/minion - - text: 'startup_states: highstate' - enable_salt_minion: service.enabled: - name: salt-minion diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 62b451bec..c9399eab4 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1523,8 +1523,12 @@ soc: saltstackDir: /opt/so/saltstack bypassEnabled: false postgres: - host: - password: + host: "" + port: 5432 + sslMode: "allow" + database: securityonion + user: "" + password: "" salt: queueDir: /opt/sensoroni/queue timeoutMs: 45000 diff --git a/salt/soc/merged.map.jinja b/salt/soc/merged.map.jinja index b34efb11d..cfc0fafbd 100644 --- a/salt/soc/merged.map.jinja +++ b/salt/soc/merged.map.jinja @@ -20,7 +20,8 @@ {% do SOCMERGED.config.server.modules.postgres.update({'host': GLOBALS.manager}) %} {% endif %} {% if not SOCMERGED.config.server.modules.postgres.password %} -{% do SOCMERGED.config.server.modules.postgres.update({'password': salt['pillar.get']('secrets:postgres_pass', '')}) %} +{% do SOCMERGED.config.server.modules.postgres.update({'password': salt['pillar.get']('postgres:auth:users:so_postgres_user:pass', '')}) %} +{% do SOCMERGED.config.server.modules.postgres.update({'user': salt['pillar.get']('postgres:auth:users:so_postgres_user:user', 'so_postgres')}) %} {% endif %} {# if SOCMERGED.config.server.modules.cases == httpcase details come from the soc pillar #} diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index 3cb244eed..b2ac6d175 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -468,8 +468,24 @@ soc: description: Hostname or IP address of the PostgreSQL server used by SOC. Defaults to the manager hostname. global: True advanced: True + port: + description: Port of the PostgreSQL server used by SOC. + global: True + advanced: True + sslMode: + description: "Use encrypted connections to the PostgreSQL server. Must be one of the following values: disable, allow, prefer, require, verify-ca, verify-full. Defaults to allow." + global: True + advanced: True + database: + description: Database used by SOC to authenticate to the PostgreSQL server. + global: True + advanced: True + user: + description: Username used by SOC to authenticate to the PostgreSQL server. + global: True + advanced: True password: - description: Password used by SOC to authenticate to the PostgreSQL server. Defaults to the postgres superuser password seeded in the secrets pillar. + description: Password used by SOC to authenticate to the PostgreSQL server. global: True sensitive: True advanced: True diff --git a/setup/so-functions b/setup/so-functions index c94b8eee7..2d5181dc1 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -539,16 +539,19 @@ configure_minion() { " x509_v2: true"\ "log_level: info"\ "log_level_logfile: info"\ - "log_file: /opt/so/log/salt/minion"\ - "#startup_states: highstate" >> "$minion_config" + "log_file: /opt/so/log/salt/minion" >> "$minion_config" } -checkin_at_boot() { - local minion_config=/etc/salt/minion +mark_setup_complete() { + # Writes the setup-complete marker. Salt's so-boot-highstate.service + # (boot-time oneshot) and the so-user_sync cron gate in + # salt/manager/sync_es_users.sls both key off this file. + local marker=/opt/so/state/setup-complete - info "Enabling checkin at boot" - sed -i 's/#startup_states: highstate/startup_states: highstate/' "$minion_config" + info "Marking setup as complete" + mkdir -p "$(dirname "$marker")" + touch "$marker" } check_requirements() { @@ -977,6 +980,8 @@ docker_seed_registry() { docker_seed_update_percent=25 update_docker_containers 'netinstall' '' 'docker_seed_update' '/dev/stdout' 2>&1 | tee -a "$setup_log" + # Use pipe exit status of 'update_docker_containers' for return code + return ${PIPESTATUS[0]} fi } diff --git a/setup/so-setup b/setup/so-setup index 6c77e781c..72cd555d6 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -767,7 +767,10 @@ if ! [[ -f $install_opt_file ]]; then title "Applying the registry state" logCmd "salt-call state.apply -l info registry" title "Seeding the docker registry" - docker_seed_registry + if ! docker_seed_registry; then + error "Failed to seed the docker registry" + fail_setup + fi title "Applying the manager state" logCmd "salt-call state.apply -l info manager" logCmd "salt-call state.apply influxdb -l info" @@ -792,7 +795,7 @@ if ! [[ -f $install_opt_file ]]; then error "Failed to run so-elastic-fleet-setup" fail_setup fi - checkin_at_boot + mark_setup_complete set_initial_firewall_access initialize_elasticsearch_indices "so-case so-casehistory so-assistant-session so-assistant-chat" # run a final highstate before enabling scheduled highstates.