diff --git a/salt/common/tools/sbin/so-restart b/salt/common/tools/sbin/so-restart index 14747d134..b3864d2d1 100755 --- a/salt/common/tools/sbin/so-restart +++ b/salt/common/tools/sbin/so-restart @@ -35,6 +35,9 @@ case $1 in "elastic-fleet"|"elasticfleet") docker_check_running "elastic-fleet" "--stop" docker rm "so-elastic-fleet" 2> /dev/null + # Removing the elastic fleet state directory, so that the next startup re-enrolls with a fresh policy + rm -rf /opt/so/conf/elastic-fleet/state + salt-call state.apply elasticfleet queue=True ;; *) diff --git a/salt/common/tools/sbin/so-status b/salt/common/tools/sbin/so-status index f4abd8aa3..b50cdff14 100755 --- a/salt/common/tools/sbin/so-status +++ b/salt/common/tools/sbin/so-status @@ -74,13 +74,13 @@ def output(options, console, code, data): summary = { "status_code": code, "containers": data } print(json.dumps(summary)) elif "-q" not in options: - if code == 2: - console.print(" [bold yellow]:hourglass: [bold white]System appears to be starting. No highstate has completed since the system was restarted.") - elif code == 99: + if code == 99: console.print(" [bold red]:exclamation: [bold white]Installation does not appear to be complete. A highstate has not fully completed.") elif code == 100: console.print(" [bold red]:exclamation: [bold white]Installation encountered errors.") else: + if code == 2: + console.print(" [bold yellow]:hourglass: [bold white]System appears to be starting. No highstate has completed since the system was restarted. Container status is shown below.") table = Table(title = "Security Onion Status", show_edge = False, safe_box = True, box = box.MINIMAL) table.add_column("Container", justify="right", style="white", no_wrap=True) table.add_column("Status", justify="left", style="green", no_wrap=True) @@ -154,8 +154,14 @@ def check_status(options, console): code = check_installation_status(options, console) if code == 0: code = check_system_status(options, console) - if code == 0: - code, container_list = check_container_status(options, console) + # Containers now start on boot without a highstate, so gather/display their + # status even when the system is still "starting" (code 2). Keep the starting + # code as the exit/status_code so SOC keeps showing the "restarting" message + # on the Grid until a highstate completes. + if code == 0 or code == 2: + container_code, container_list = check_container_status(options, console) + if code == 0: + code = container_code output(options, console, code, container_list) return code @@ -180,4 +186,3 @@ def main(): if __name__ == "__main__": main() - diff --git a/salt/common/tools/sbin/so-stop b/salt/common/tools/sbin/so-stop index d036a7b63..9218b44f7 100755 --- a/salt/common/tools/sbin/so-stop +++ b/salt/common/tools/sbin/so-stop @@ -29,6 +29,8 @@ case $1 in "elasticfleet"|"elastic-fleet") docker_check_running "elastic-fleet" "--stop" docker rm "so-elastic-fleet" 2> /dev/null + # Removing the elastic fleet state directory, so that the next startup re-enrolls with a fresh policy + rm -rf /opt/so/conf/elastic-fleet/state ;; *) docker_check_running "$1" "--stop" diff --git a/salt/common/tools/sbin_jinja/so-salt-minion-check b/salt/common/tools/sbin_jinja/so-salt-minion-check index 1376193cc..c78846a5f 100755 --- a/salt/common/tools/sbin_jinja/so-salt-minion-check +++ b/salt/common/tools/sbin_jinja/so-salt-minion-check @@ -18,6 +18,7 @@ QUIET=false UPTIME_REQ=1800 #in seconds, how long the box has to be up before considering restarting salt-minion due to /opt/so/log/salt/state-apply-test not being touched +HIGHSTATE_UPTIME_REQ=900 #in seconds; if the box has been up this long and no highstate has completed since boot, force one CURRENT_TIME=$(date +%s) SYSTEM_START_TIME=$(date -d "$(> "/opt/so/log/salt/so-salt-minion-check" 2>&1 & + RESTARTED=true else - log "/opt/so/log/salt/healthcheck-state-apply last touched: `date -d @$LAST_HEALTHCHECK_STATE_APPLY` must be touched by `date -d @$THRESHOLD_DATE` to avoid salt-minion restart" I + log "[minion-restart-check] healthy: state-apply-test last touched `date -d @$LAST_HEALTHCHECK_STATE_APPLY`, must go stale past `date -d @$THRESHOLD_DATE` to trigger a salt-minion restart" I fi else - log "system uptime only $((CURRENT_TIME-SYSTEM_START_TIME)) seconds does not meet $UPTIME_REQ second requirement." I + log "[minion-restart-check] skipped: system uptime $((CURRENT_TIME-SYSTEM_START_TIME))s is below the ${UPTIME_REQ}s minimum required before a salt-minion restart" I +fi + +# Check 2 (boot-highstate-check): if the host has been up long enough but no highstate +# has completed since this boot, force one. This recovers a host whose boot highstate +# (so-boot-highstate.service) failed or was skipped, even while the minion is otherwise +# healthy (touching state-apply-test). We deliberately do NOT enable highstate here: if +# soup has disabled it during an upgrade, Salt will refuse the highstate and we avoid +# forcing one mid-upgrade. +if $RESTARTED; then + log "[boot-highstate-check] skipped: minion-restart-check already queued a highstate this run" I +elif [ $CURRENT_TIME -lt $((SYSTEM_START_TIME+HIGHSTATE_UPTIME_REQ)) ]; then + log "[boot-highstate-check] skipped: system uptime $((CURRENT_TIME-SYSTEM_START_TIME))s is below the ${HIGHSTATE_UPTIME_REQ}s minimum required before forcing a highstate" I +elif [ $LAST_HIGHSTATE_END -ge $SYSTEM_START_TIME ]; then + log "[boot-highstate-check] healthy: a highstate completed at `date -d @$LAST_HIGHSTATE_END`, after this boot at `date -d @$SYSTEM_START_TIME`" I +elif salt-call --local saltutil.running 2>/dev/null | grep -q 'state.highstate'; then + log "[boot-highstate-check] no highstate has completed since boot, but one is already running; skipping" I +else + log "[boot-highstate-check] no highstate has completed since boot after $((CURRENT_TIME-SYSTEM_START_TIME))s uptime; applying highstate" E + nohup bash -c 'salt-call state.highstate -l info queue=True' >> "/opt/so/log/salt/so-salt-minion-check" 2>&1 & fi diff --git a/salt/elasticfleet/defaults.yaml b/salt/elasticfleet/defaults.yaml index 022600083..a3132d3f4 100644 --- a/salt/elasticfleet/defaults.yaml +++ b/salt/elasticfleet/defaults.yaml @@ -1,6 +1,5 @@ elasticfleet: enabled: False - patch_version: 9.3.3+build202604082258 # Elastic Agent specific patch release. enable_manager_output: True config: server: diff --git a/salt/elasticfleet/enabled.sls b/salt/elasticfleet/enabled.sls index feb83fc2f..d9694f07d 100644 --- a/salt/elasticfleet/enabled.sls +++ b/salt/elasticfleet/enabled.sls @@ -11,6 +11,10 @@ {# This value is generated during node install and stored in minion pillar #} {% set SERVICETOKEN = salt['pillar.get']('elasticfleet:config:server:es_token','') %} +{# Prevent Elastic Agent from re-enrolling with a new agent.id everytime the container starts up. + - if a fresh enrollment is needed use 'so-stop elasticfleet' + #} +{% set ENROLLED = salt['file.file_exists']('/opt/so/conf/elastic-fleet/state/fleet.enc') %} include: - ca @@ -66,6 +70,7 @@ so-elastic-fleet: - /etc/pki/elasticfleet-server.crt:/etc/pki/elasticfleet-server.crt:ro - /etc/pki/elasticfleet-server.key:/etc/pki/elasticfleet-server.key:ro - /etc/pki/tls/certs/intca.crt:/etc/pki/tls/certs/intca.crt:ro + - /opt/so/conf/elastic-fleet/state:/usr/share/elastic-agent/state - /opt/so/log/elasticfleet:/usr/share/elastic-agent/logs {% if DOCKERMERGED.containers['so-elastic-fleet'].custom_bind_mounts %} {% for BIND in DOCKERMERGED.containers['so-elastic-fleet'].custom_bind_mounts %} @@ -73,6 +78,7 @@ so-elastic-fleet: {% endfor %} {% endif %} - environment: + {% if not ENROLLED %} - FLEET_SERVER_ENABLE=true - FLEET_URL=https://{{ GLOBALS.hostname }}:8220 - FLEET_SERVER_ELASTICSEARCH_HOST=https://{{ GLOBALS.manager }}:9200 @@ -82,6 +88,9 @@ so-elastic-fleet: - FLEET_SERVER_CERT_KEY=/etc/pki/elasticfleet-server.key - FLEET_CA=/etc/pki/tls/certs/intca.crt - FLEET_SERVER_ELASTICSEARCH_CA=/etc/pki/tls/certs/intca.crt + {% endif %} + - STATE_PATH=/usr/share/elastic-agent/state + - CONFIG_PATH=/usr/share/elastic-agent/state - LOGS_PATH=logs {% if DOCKERMERGED.containers['so-elastic-fleet'].extra_env %} {% for XTRAENV in DOCKERMERGED.containers['so-elastic-fleet'].extra_env %} @@ -100,6 +109,7 @@ so-elastic-fleet: - x509: etc_elasticfleet_crt - require: - file: trusttheca + - file: eastatedir - x509: etc_elasticfleet_key - x509: etc_elasticfleet_crt diff --git a/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json b/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json index c27da26f7..7d64fd1ab 100644 --- a/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json +++ b/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json @@ -5,7 +5,7 @@ "package": { "name": "endpoint", "title": "Elastic Defend", - "version": "9.3.0", + "version": "9.3.1", "requires_root": true }, "enabled": true, diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json index 32d210172..be05d59e0 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json @@ -29,7 +29,7 @@ "\\.gz$" ], "include_files": [], - "processors": "- dissect:\n tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"\n- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n- drop_fields:\n fields: [\"host\"]\n ignore_missing: true\n- add_fields:\n target: data_stream\n fields:\n type: logs\n dataset: system.security\n- add_fields:\n target: event\n fields:\n dataset: system.security\n module: system\n imported: true\n- add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.security-2.15.0\n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.sysmon_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.sysmon_operational\n module: windows\n imported: true\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.sysmon_operational-3.8.0\n- if:\n equals:\n winlog.channel: 'Application'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.application\n - add_fields:\n target: event\n fields:\n dataset: system.application\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.application-2.15.0\n- if:\n equals:\n winlog.channel: 'System'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.system\n - add_fields:\n target: event\n fields:\n dataset: system.system\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.system-2.15.0\n \n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.powershell_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.powershell_operational\n module: windows\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.powershell_operational-3.8.0\n- add_fields:\n target: data_stream\n fields:\n dataset: import", + "processors": "- dissect:\n tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"\n- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n- drop_fields:\n fields: [\"host\"]\n ignore_missing: true\n- add_fields:\n target: data_stream\n fields:\n type: logs\n dataset: system.security\n- add_fields:\n target: event\n fields:\n dataset: system.security\n module: system\n imported: true\n- add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.security-2.20.0\n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.sysmon_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.sysmon_operational\n module: windows\n imported: true\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.sysmon_operational-3.8.3\n- if:\n equals:\n winlog.channel: 'Application'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.application\n - add_fields:\n target: event\n fields:\n dataset: system.application\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.application-2.20.0\n- if:\n equals:\n winlog.channel: 'System'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.system\n - add_fields:\n target: event\n fields:\n dataset: system.system\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.system-2.20.0\n \n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.powershell_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.powershell_operational\n module: windows\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.powershell_operational-3.8.3\n- add_fields:\n target: data_stream\n fields:\n dataset: import", "tags": [ "import" ], diff --git a/salt/elasticfleet/install_agent_grid.sls b/salt/elasticfleet/install_agent_grid.sls index 482af2e1e..7ed727b48 100644 --- a/salt/elasticfleet/install_agent_grid.sls +++ b/salt/elasticfleet/install_agent_grid.sls @@ -10,6 +10,15 @@ {% set AGENT_STATUS = salt['service.available']('elastic-agent') %} {% set AGENT_EXISTS = salt['file.file_exists']('/opt/Elastic/Agent/elastic-agent') %} +so-elastic-agent-install: + file.managed: + - name: /usr/sbin/so-elastic-agent-install + - source: salt://elasticfleet/tools/sbin/so-elastic-agent-install + - user: 947 + - group: 939 + - mode: 755 + - show_changes: False + {% if not AGENT_STATUS or not AGENT_EXISTS %} pull_agent_installer: @@ -21,11 +30,9 @@ pull_agent_installer: run_installer: cmd.run: - - name: ./so-elastic-agent_linux_amd64 -token={{ GRIDNODETOKEN }} -force - - cwd: /opt/so - - retry: - attempts: 3 - interval: 20 + - name: /usr/sbin/so-elastic-agent-install "{{ GRIDNODETOKEN }}" + - require: + - file: pull_agent_installer cleanup_agent_installer: file.absent: diff --git a/salt/elasticfleet/tools/sbin/so-elastic-agent-install b/salt/elasticfleet/tools/sbin/so-elastic-agent-install new file mode 100644 index 000000000..8f57d217c --- /dev/null +++ b/salt/elasticfleet/tools/sbin/so-elastic-agent-install @@ -0,0 +1,100 @@ +#!/bin/bash +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +. /usr/sbin/so-elastic-fleet-common + + +# passed in as arg from elasticfleet/install_agent_grid.sls, else pulled from pillar later +GRIDNODETOKEN="$1" +LOGFILE="/opt/so/SO-Elastic-Agent_Installer_Health.log" + +check_agent_health() { + timeout=300 + interval=10 + start=$SECONDS + + while (( SECONDS - start < timeout )); do + agent_status=$(elastic-agent status 2>&1) + echo -e "\n$(date)\n$agent_status\n" >> "$LOGFILE" + if echo "$agent_status" | grep -A1 'elastic-agent$' | grep -q 'status: (HEALTHY)'; then + return 0 + fi + echo "The Elastic Agent is not yet healthy. Waiting for ${interval} seconds before checking again..." + sleep "$interval" + done + + echo "The Elastic Agent did not become healthy within ${timeout} seconds" + return 1 +} + +uninstall_agent() { + if command -v elastic-agent >/dev/null 2>&1; then + elastic-agent uninstall -f + fi +} + +if [[ -z "$GRIDNODETOKEN" ]]; then + noderole=$(so-yaml.py get -r /etc/salt/grains role) + if [[ "$noderole" == "so-heavynode" ]]; then + GRIDNODETOKEN=$(salt-call pillar.get global:fleet_grid_enrollment_token_heavy --out=newline_values_only) + else + GRIDNODETOKEN=$(salt-call pillar.get global:fleet_grid_enrollment_token_general --out=newline_values_only) + fi +fi + +if [[ -z "$GRIDNODETOKEN" ]]; then + echo "Unable to determine Elastic Fleet enrollment token. Exiting." + + exit 1 +fi + +if [[ ! -x /opt/so/so-elastic-agent_linux_amd64 ]]; then + echo "Downloading so-elastic-agent installer... This could take a while if another Salt job is running." + + # When running outside of elasticfleet/install_agent_grid.sls we need to download the installer independently. + # PYTHONWARNINGS="ignore" to avoid messages like the following when running salt-call: + # '/opt/saltstack/salt/lib/python3.10/site-packages/salt/transport/base.py:129: TransportWarning: Unclosed transport! + # File "/bin/salt-call", line 12, in + # sys.exit(salt_call())' + + PYTHONWARNINGS="ignore" salt-call state.single file.managed name=/opt/so/so-elastic-agent_linux_amd64 source=salt://elasticfleet/files/so_agent-installers/so-elastic-agent_linux_amd64 mode=755 makedirs=True queue=True + +fi + +if [[ -x /opt/so/so-elastic-agent_linux_amd64 ]]; then + attempts=0 + cd /opt/so/ || exit 1 + + truncate -s 0 "$LOGFILE" + + uninstall_agent + + while [[ $attempts -lt 3 ]]; do + if ./so-elastic-agent_linux_amd64 -token="$GRIDNODETOKEN" -force && echo "Verifying Elastic Agent health..." && check_agent_health; then + rm -f /opt/so/so-elastic-agent_linux_amd64 + elastic-agent status + + exit 0 + fi + + attempts=$((attempts + 1)) + + if [[ $attempts -lt 3 ]]; then + echo "Unable to verify Elastic Agent health... Retrying in 20 seconds..." + sleep 20 + fi + done + + uninstall_agent + rm -f /opt/so/so-elastic-agent_linux_amd64 + echo "The so-elastic-agent installer failed after 3 attempts. Exiting." + + exit 1 +else + echo "Unable to locate so-elastic-agent installer. Exiting." + + exit 1 +fi diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers index d25c18e29..de342657f 100755 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers @@ -30,7 +30,7 @@ done if [[ -z $FLEETHOST ]] || [[ -z $ENROLLMENTOKEN ]]; then printf "\nFleet Host URL, Enrollment Token or Elastic Version empty - exiting..." printf "\nFleet Host: $FLEETHOST, Enrollment Token: $ENROLLMENTOKEN\n" - exit + exit 1 fi OSARCH=( "linux-x86_64" "windows-x86_64" "darwin-x86_64" "darwin-aarch64" ) @@ -62,31 +62,54 @@ do done GOTARGETOS=( "linux" "windows" "darwin" "darwin/arm64" ) -GOARCH="amd64" printf "\n### Generating OS packages using the cleaned up tarballs" -for GOOS in "${GOTARGETOS[@]}" -do +for GOOS in "${GOTARGETOS[@]}"; do + GOARCH="amd64" if [[ $GOOS == 'darwin/arm64' ]]; then GOOS="darwin" && GOARCH="arm64"; fi printf "\n\n### Generating $GOOS/$GOARCH Installer...\n" docker run -e CGO_ENABLED=0 -e GOOS=$GOOS -e GOARCH=$GOARCH \ --mount type=bind,source=/etc/pki/tls/certs/,target=/workspace/files/cert/ \ --mount type=bind,source=/nsm/elastic-agent-workspace/,target=/workspace/files/elastic-agent/ \ - --mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/,target=/output/ \ + --mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/,target=/output/ \ {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} go build -ldflags "-X main.fleetHostURLsList=$FLEETHOST -X main.enrollmentToken=$ENROLLMENTOKEN" -o /output/so-elastic-agent_${GOOS}_${GOARCH} printf "\n### $GOOS/$GOARCH Installer Generated...\n" done printf "\n\n### Generating MSI...\n" -cp /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/so-elastic-agent_windows_amd64 /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/so-elastic-agent_windows_amd64.exe +cp /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_windows_amd64 /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_windows_amd64.exe docker run \ ---mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/,target=/output/ -w /output \ +--mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/,target=/output/ -w /output \ {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} wixl -o so-elastic-agent_windows_amd64_msi --arch x64 /workspace/so-elastic-agent.wxs printf "\n### MSI Generated...\n" +# Verify installers were created +for GOOS in "${GOTARGETOS[@]}"; do + GOARCH="amd64" + if [[ $GOOS == 'darwin/arm64' ]]; then GOOS="darwin"; GOARCH="arm64"; fi + if [[ ! -f /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_${GOOS}_${GOARCH} ]]; then + printf "\n### ERROR: Installer for %s/%s was not generated. Exiting...\n" "$GOOS" "$GOARCH" + exit 1 + fi + # After verifying new installer was generated, move it to so_agent-installers directory + mv /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_${GOOS}_${GOARCH} /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/ +done + +# Verify MSI installer +if [[ ! -f /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_windows_amd64_msi ]]; then + printf "\n### ERROR: Installer MSI was not generated. Exiting...\n" + exit 1 +else + # After verifying new installer MSI was generated, move it to so_agent-installers directory + mv /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_windows_amd64_msi /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/ +fi + printf "\n### Cleaning up temp files \n" rm -rf /nsm/elastic-agent-workspace -rm -rf /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/so-elastic-agent_windows_amd64.exe +rm -rf /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_windows_amd64.exe printf "\n### Copying so_agent-installers to /nsm/elastic-fleet/ for nginx.\n" \cp -vr /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/ /nsm/elastic-fleet/ chmod 644 /nsm/elastic-fleet/so_agent-installers/* + +# if we got here all installers have been generated successfully +exit 0 diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup index b62310375..22e0c7554 100755 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup @@ -244,11 +244,37 @@ printf '%s\n'\ "" >> "$global_pillar_file" # Call Elastic-Fleet Salt State -printf "\nApplying elasticfleet state" -salt-call state.apply elasticfleet queue=True +printf "\nApplying elasticfleet state\n" +for state_attempt in {1..3}; do + if salt-call state.apply elasticfleet queue=True; then + break + elif [[ $state_attempt -lt 3 ]]; then + printf "\nElasticfleet state did not complete successfully... Attempt (%s/3). Retrying...\n" "$state_attempt" + sleep 10 + else + printf "\nFailure(s) in elasticfleet state... Exiting...\n" + exit 1 + fi +done +printf "\nRunning so-elastic-agent-gen-installers\n" # Generate installers & install Elastic Agent on the node -so-elastic-agent-gen-installers -printf "\nApplying elasticfleet.install_agent_grid state" -salt-call state.apply elasticfleet.install_agent_grid queue=True -exit 0 +for agent_gen_attempt in {1..3}; do + if so-elastic-agent-gen-installers; then + break + elif [[ $agent_gen_attempt -lt 3 ]]; then + printf "\nUnable to generate Elastic Agent installers... Attempt (%s/3). Retrying...\n" "$agent_gen_attempt" + sleep 10 + else + printf "\nFailed to generate Elastic Agent installers after 3 attempts. Exiting...\n" + exit 1 + fi +done + +printf "\nApplying elasticfleet.install_agent_grid state\n" +if ! salt-call state.apply elasticfleet.install_agent_grid queue=True; then + printf "\nFailure(s) in elasticfleet.install_agent_grid state... Exiting...\n" + exit 1 +fi + +printf "\nElastic Fleet setup completed successfully\n" \ No newline at end of file diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index f0b01b3ca..14d753c21 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -1,6 +1,6 @@ elasticsearch: enabled: false - version: 9.3.3 + version: 9.3.7 index_clean: true data_retention_method: DLM vm: diff --git a/salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.2 b/salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.4 similarity index 96% rename from salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.2 rename to salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.4 index 1ea828514..c01383fde 100644 --- a/salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.2 +++ b/salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.4 @@ -118,70 +118,70 @@ { "pipeline": { "tag": "pipeline_e16851a7", - "name": "logs-pfsense.log-1.25.2-firewall", + "name": "logs-pfsense.log-1.25.4-firewall", "if": "ctx.event.provider == 'filterlog'" } }, { "pipeline": { "tag": "pipeline_828590b5", - "name": "logs-pfsense.log-1.25.2-openvpn", + "name": "logs-pfsense.log-1.25.4-openvpn", "if": "ctx.event.provider == 'openvpn'" } }, { "pipeline": { "tag": "pipeline_9d37039c", - "name": "logs-pfsense.log-1.25.2-ipsec", + "name": "logs-pfsense.log-1.25.4-ipsec", "if": "ctx.event.provider == 'charon'" } }, { "pipeline": { "tag": "pipeline_ad56bbca", - "name": "logs-pfsense.log-1.25.2-dhcp", + "name": "logs-pfsense.log-1.25.4-dhcp", "if": "[\"dhcpd\", \"dhclient\", \"dhcp6c\", \"dnsmasq-dhcp\"].contains(ctx.event.provider)" } }, { "pipeline": { "tag": "pipeline_dd85553d", - "name": "logs-pfsense.log-1.25.2-unbound", + "name": "logs-pfsense.log-1.25.4-unbound", "if": "ctx.event.provider == 'unbound'" } }, { "pipeline": { "tag": "pipeline_720ed255", - "name": "logs-pfsense.log-1.25.2-haproxy", + "name": "logs-pfsense.log-1.25.4-haproxy", "if": "ctx.event.provider == 'haproxy'" } }, { "pipeline": { "tag": "pipeline_456beba5", - "name": "logs-pfsense.log-1.25.2-php-fpm", + "name": "logs-pfsense.log-1.25.4-php-fpm", "if": "ctx.event.provider == 'php-fpm'" } }, { "pipeline": { "tag": "pipeline_a0d89375", - "name": "logs-pfsense.log-1.25.2-squid", + "name": "logs-pfsense.log-1.25.4-squid", "if": "ctx.event.provider == 'squid'" } }, { "pipeline": { "tag": "pipeline_c2f1ed55", - "name": "logs-pfsense.log-1.25.2-snort", + "name": "logs-pfsense.log-1.25.4-snort", "if": "ctx.event.provider == 'snort'" } }, { "pipeline": { "tag":"pipeline_33db1c9e", - "name": "logs-pfsense.log-1.25.2-suricata", + "name": "logs-pfsense.log-1.25.4-suricata", "if": "ctx.event.provider == 'suricata'" } }, diff --git a/salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.2-suricata b/salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.4-suricata similarity index 100% rename from salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.2-suricata rename to salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.4-suricata diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index 46e836c9c..95880bb9d 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -28,14 +28,14 @@ elasticsearch: description: The maximum number of memory map areas a process may use. Elasticsearch uses a mmapfs directory by default to store its indices. The default operating system limits on mmap counts could be too low, which may result in out of memory exceptions. forcedType: int helpLink: elasticsearch - retention: + retention: retention_pct: decription: Total percentage of space used by Elasticsearch for multi node clusters helpLink: elasticsearch global: True config: cluster: - name: + name: description: The name of the Security Onion Elasticsearch cluster, for identification purposes. readonly: True global: True @@ -55,13 +55,13 @@ elasticsearch: forcedType: bool helpLink: elasticsearch watermark: - low: + low: description: The lower percentage of used disk space representing a healthy node. helpLink: elasticsearch - high: + high: description: The higher percentage of used disk space representing an unhealthy node. helpLink: elasticsearch - flood_stage: + flood_stage: description: The max percentage of used disk space that will cause the node to take protective actions, such as blocking incoming events. helpLink: elasticsearch action: @@ -172,11 +172,11 @@ elasticsearch: forcedType: int global: True helpLink: elasticsearch - refresh_interval: + refresh_interval: description: Seconds between index refreshes. Shorter intervals can cause query performance to suffer since this is a synchronous and resource-intensive operation. global: True helpLink: elasticsearch - number_of_shards: + number_of_shards: description: Number of shards required for this index. Using multiple shards increases fault tolerance, but also increases storage and network costs. global: True helpLink: elasticsearch @@ -269,7 +269,7 @@ elasticsearch: global: True advanced: True warm: - min_age: + min_age: description: Minimum age of index. ex. 30d - This determines when the index should be moved to the warm tier. Nodes in the warm tier generally don’t need to be as fast as those in the hot tier. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and warm min_age set to 30 then there will be 30 days from index creation to rollover and then an additional 30 days before moving to warm tier. regex: ^[0-9]{1,5}d$ forcedType: string @@ -370,7 +370,7 @@ elasticsearch: template: settings: index: - number_of_replicas: + number_of_replicas: description: Number of replicas required for this index. Multiple replicas protects against data loss, but also increases storage costs. forcedType: int global: True @@ -391,12 +391,12 @@ elasticsearch: global: True advanced: True helpLink: elasticsearch - refresh_interval: + refresh_interval: description: Seconds between index refreshes. Shorter intervals can cause query performance to suffer since this is a synchronous and resource-intensive operation. global: True advanced: True helpLink: elasticsearch - number_of_shards: + number_of_shards: description: Number of shards required for this index. Using multiple shards increases fault tolerance, but also increases storage and network costs. global: True advanced: True @@ -645,6 +645,7 @@ elasticsearch: global: True advanced: True helpLink: elasticsearch + so-logs-soc: *dataStreamSettings so-logs-system_x_auth: *dataStreamSettings so-logs-system_x_syslog: *dataStreamSettings so-logs-system_x_system: *dataStreamSettings diff --git a/salt/hydra/enabled.sls b/salt/hydra/enabled.sls index b19f9a80d..74106f550 100644 --- a/salt/hydra/enabled.sls +++ b/salt/hydra/enabled.sls @@ -22,6 +22,7 @@ include: so-hydra: docker_container.running: - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-hydra:{{ GLOBALS.so_version }} + - restart_policy: unless-stopped - hostname: hydra - name: so-hydra - networks: @@ -58,8 +59,6 @@ so-hydra: - {{ ULIMIT.name }}={{ ULIMIT.soft }}:{{ ULIMIT.hard }} {% endfor %} {% endif %} - # Intentionally unless-stopped -- matches the fleet default. - - restart_policy: unless-stopped - watch: - file: hydraconfig - require: diff --git a/salt/kibana/defaults.yaml b/salt/kibana/defaults.yaml index ecf56756b..2cf6fe92f 100644 --- a/salt/kibana/defaults.yaml +++ b/salt/kibana/defaults.yaml @@ -22,7 +22,7 @@ kibana: - default - file migrations: - discardCorruptObjects: "9.3.3" + discardCorruptObjects: "9.3.7" telemetry: enabled: False xpack: diff --git a/salt/kratos/enabled.sls b/salt/kratos/enabled.sls index 7dd13732e..ab13d759f 100644 --- a/salt/kratos/enabled.sls +++ b/salt/kratos/enabled.sls @@ -15,6 +15,7 @@ include: so-kratos: docker_container.running: - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-kratos:{{ GLOBALS.so_version }} + - restart_policy: unless-stopped - hostname: kratos - name: so-kratos - networks: @@ -51,8 +52,6 @@ so-kratos: - {{ ULIMIT.name }}={{ ULIMIT.soft }}:{{ ULIMIT.hard }} {% endfor %} {% endif %} - # Intentionally unless-stopped -- matches the fleet default. - - restart_policy: unless-stopped - watch: - file: kratosschema - file: kratosconfig diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 566d1482e..65141c9a9 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -12,7 +12,17 @@ UPDATE_DIR=/tmp/sogh/securityonion DEFAULT_SALT_DIR=/opt/so/saltstack/default INSTALLEDVERSION=$(cat /etc/soversion) -POSTVERSION=$INSTALLEDVERSION +# /etc/sopostversion is a soup-owned marker (no salt state manages it) tracking how +# far the post-upgrade walk has progressed. Its presence means a prior upgrade did +# not finish its post-upgrade steps; its contents are the resume point. It is read +# here before preupgrade_changes mutates INSTALLEDVERSION and before any highstate +# stamps /etc/soversion from the pillar. +POSTVERSION_FILE=/etc/sopostversion +if [ -f "$POSTVERSION_FILE" ]; then + POSTVERSION=$(cat "$POSTVERSION_FILE") +else + POSTVERSION=$INSTALLEDVERSION +fi INSTALLEDSALTVERSION=$(salt --versions-report | grep Salt: | awk '{print $2}') BATCHSIZE=5 SOUP_LOG=/root/soup.log @@ -23,6 +33,10 @@ NOTIFYCUSTOMELASTICCONFIG=false TOPFILE=/opt/so/saltstack/default/salt/top.sls BACKUPTOPFILE=/opt/so/saltstack/default/salt/top.sls.backup SALTUPGRADED=false +# Set true once soup begins modifying the system (past the pre-flight checks), so the +# EXIT trap can tell the user the update did not finish and must be re-run. Only the +# pre-flight gates (ES compatibility, disk, network) fail before this is set. +SOUP_UPGRADE_STARTED=false SALT_CLOUD_INSTALLED=false SALT_CLOUD_CONFIGURED=false # Check if salt-cloud is installed @@ -123,6 +137,28 @@ check_err() { echo "SOUP XTRACE debug log (if enabled) at $SOUP_DEBUG_LOG. Re-run soup with SOUP_DEBUG=1 to create $SOUP_DEBUG_LOG" + # If soup had already started modifying the system, make it unmistakable that the + # update is incomplete and must be re-run. soup is resumable: a version upgrade + # picks up from the /etc/sopostversion marker, and a hotfix re-applies because + # /etc/sohotfix is only advanced after a successful highstate. + if [[ "$SOUP_UPGRADE_STARTED" == "true" ]]; then + echo "" + echo "==============================================================================" + echo " UPGRADE INCOMPLETE" + echo "==============================================================================" + echo " This soup run did NOT finish. Your Security Onion installation may be in a" + echo " partially-updated state and is not yet fully upgraded." + echo "" + echo " Review the error above and $SOUP_LOG, resolve the underlying problem, then" + echo " run soup again to resume and complete the update:" + echo "" + echo " sudo soup" + echo "" + echo " soup is resumable -- re-running it continues from where this run stopped." + echo "==============================================================================" + echo "" + fi + exit $exit_code fi @@ -291,6 +327,30 @@ check_pillar_items() { fi } +check_cluster_health() { + echo "Checking Elasticsearch cluster health." + # Require a 'green' cluster before upgrading; anything less (yellow, red, or + # unreachable) blocks. Modeled on the wait used in so-elasticsearch-roles-load. + if so-elasticsearch-query "_cluster/health?wait_for_status=green&timeout=120s" --fail > /dev/null 2>&1; then + printf "\nThe Elasticsearch cluster is healthy (green). We can proceed with SOUP.\n\n" + else + printf "\nThe Elasticsearch cluster is not green. Please resolve the cluster health issue so the cluster is green before running SOUP again.\n\n" + exit 0 + fi +} + +check_fleet_server() { + echo "Checking that Elastic Fleet Server is responding." + # Modeled on the wait_for_so-elastic-fleet state check in elasticfleet/enabled.sls, + # which waits for HTTP 200 from the Fleet Server status API. + if curl -sk --fail --retry 3 --retry-delay 10 --max-time 30 "https://localhost:8220/api/status" > /dev/null 2>&1; then + printf "\nElastic Fleet Server is responding. We can proceed with SOUP.\n\n" + else + printf "\nElastic Fleet Server is not responding at https://localhost:8220/api/status. Please ensure Elastic Fleet is healthy before running SOUP again.\n\n" + exit 0 + fi +} + check_saltmaster_status() { set +e echo "Waiting on the Salt Master service to be ready." @@ -414,6 +474,13 @@ preupgrade_changes() { true } +set_postversion() { + # Persist post-upgrade walk progress so an interrupted upgrade can resume the + # remaining steps on the next soup run (see /etc/sopostversion handling). + POSTVERSION="$1" + echo "$POSTVERSION" > "$POSTVERSION_FILE" +} + postupgrade_changes() { # This function is to add any new pillar items if needed. echo "Running post upgrade processes." @@ -421,6 +488,8 @@ postupgrade_changes() { [[ "$POSTVERSION" =~ ^2\.4\.21[0-9]+$ ]] && post_to_3.0.0 [[ "$POSTVERSION" == "3.0.0" ]] && post_to_3.1.0 [[ "$POSTVERSION" == "3.1.0" ]] && post_to_3.2.0 + # All applicable post-upgrade steps completed; clear the resume marker. + rm -f "$POSTVERSION_FILE" true } @@ -513,7 +582,7 @@ post_to_3.0.0() { # convert yes/no in suricata pillars to true/false convert_suricata_yes_no - POSTVERSION=3.0.0 + set_postversion 3.0.0 } ### 3.0.0 End ### @@ -692,7 +761,7 @@ ensure_postgres_local_pillar() { } ensure_salt_local_pillar() { - # The salt.auto_apply settings (moved from global.push) are a new SOC settings + # The salt.auto_apply settings are a new SOC settings # module, so the new pillar/top.sls references salt.soc_salt / salt.adv_salt # unconditionally. Managers upgrading from before this change have no # /opt/so/saltstack/local/pillar/salt/ (make_some_dirs only runs at install @@ -755,7 +824,6 @@ fix_logstash_0013_lumberjack_pipeline_name() { up_to_3.1.0() { ensure_postgres_local_pillar ensure_postgres_secret - determine_elastic_agent_upgrade elasticsearch_backup_index_templates # Clear existing component template state file. rm -f /opt/so/state/esfleet_component_templates.json @@ -792,19 +860,30 @@ post_to_3.1.0() { # Check for unhealthy / unauthorized integration transform jobs and attempt reauthorizations check_transform_health_and_reauthorize || true - POSTVERSION=3.1.0 + set_postversion 3.1.0 } ### 3.1.0 End ### ### 3.2.0 Scripts ### +recollate_postgres() { + echo "" + echo "Recollating PostgreSQL databases. The following output may contain warnings about a version mismatch, followed by a note indicating that the collation version has been changed." + for db in postgres securityonion so_telegraf; do + docker exec so-postgres psql -U postgres $db -c "reindex database $db" + docker exec so-postgres psql -U postgres $db -c "alter database $db refresh collation version" + done + echo "Recollating PostgreSQL databases complete." + echo "" +} + bootstrap_so_soc_database() { # init-db.sh is mounted into so-postgres at /docker-entrypoint-initdb.d/init-db.sh # and runs automatically only on a fresh data directory. Hosts upgrading from # 3.1.0 already have /nsm/postgres populated, so the so_soc bootstrap block # added in 3.2 never fires. Re-run the script explicitly; it's idempotent. - echo "Bootstrapping so_soc database via init-db.sh." + echo "Bootstrapping database via init-db.sh." # The postgres image has no USER directive, so `docker exec` defaults to # root, and the container env intentionally omits POSTGRES_USER (the upstream # entrypoint defaults it transiently during first-init only). Recreate both @@ -815,10 +894,13 @@ bootstrap_so_soc_database() { return 0 fi if ! $exec_cmd; then - FINAL_MESSAGE_QUEUE+=("WARNING: init-db.sh failed inside so-postgres during the 3.2.0 upgrade; the so_soc database may not have been bootstrapped. Re-run manually: $exec_cmd") + FINAL_MESSAGE_QUEUE+=("WARNING: init-db.sh failed inside so-postgres during the 3.2.0 upgrade; the database may not have been bootstrapped. Re-run manually: $exec_cmd") return 0 fi - echo "so_soc bootstrap complete." + echo "Database bootstrap complete." + + echo "Restarting so-soc container to pick up database changes" + docker restart so-soc } # Existing grids should keep ILM unless an admin explicitly opts in to DLM. @@ -891,6 +973,9 @@ update_kafka_metadata() { up_to_3.2.0() { ensure_salt_local_pillar + # download 9.3.7 elastic agent packages + determine_elastic_agent_upgrade + fix_logstash_0013_lumberjack_pipeline_name pin_elasticsearch_data_retention_method @@ -899,9 +984,12 @@ up_to_3.2.0() { } post_to_3.2.0() { + # Recollate due to image OS rebase + recollate_postgres + bootstrap_so_soc_database - # Including agent regen script here since it was missed in post_to_3.1.0 + # Generate 9.3.7 elastic agent installers echo "Regenerating Elastic Agent Installers" /sbin/so-elastic-agent-gen-installers @@ -909,7 +997,7 @@ post_to_3.2.0() { update_kafka_metadata "4.3" - POSTVERSION=3.2.0 + set_postversion 3.2.0 } ### 3.2.0 End ### @@ -1061,8 +1149,20 @@ upgrade_check() { fi [[ -f /etc/sohotfix ]] && CURRENTHOTFIX=$(cat /etc/sohotfix) if [ "$INSTALLEDVERSION" == "$NEWVERSION" ]; then + # A leftover post-version marker means a previous upgrade to this version + # advanced /etc/soversion (the highstate stamps it from the pillar) but did not + # finish its post-upgrade steps. Resume the upgrade instead of reporting "latest". + if [ -f "$POSTVERSION_FILE" ] && [ "$(cat "$POSTVERSION_FILE")" != "$NEWVERSION" ]; then + echo "A previous upgrade to $NEWVERSION did not complete its post-upgrade steps; resuming." + is_hotfix=false + return 0 + fi echo "Checking to see if there are hotfixes needed" if [ "$HOTFIXVERSION" == "$CURRENTHOTFIX" ]; then + # Reaching here means we are at the target version and NOT resuming (the resume + # check above returned otherwise). Clear any stale resume marker so a completed + # upgrade is never mistaken for a partial one and re-run on a later invocation. + rm -f "$POSTVERSION_FILE" echo "You are already running the latest version of Security Onion." exit 0 else @@ -1174,7 +1274,8 @@ verify_es_version_compatibility() { ["8.18.4"]="8.18.6 8.18.8 9.0.8" ["8.18.6"]="8.18.8 9.0.8" ["8.18.8"]="9.0.8" - ["9.0.8"]="9.3.3" + ["9.0.8"]="9.3.3 9.3.7" + ["9.3.3"]="9.3.7" ) # Elasticsearch MUST upgrade through these versions @@ -1757,6 +1858,15 @@ main() { set_minionid MINION_ROLE=$(lookup_role) echo "Found that Security Onion $INSTALLEDVERSION is currently installed." + # /etc/soversion is stamped to the target version before the upgrade fully + # completes, so a lingering resume marker means this grid is only partially + # upgraded even though the line above shows the target version. Make that explicit + # so it is not mistaken for a finished upgrade. + if [ -f "$POSTVERSION_FILE" ] && [ "$(cat "$POSTVERSION_FILE")" != "$INSTALLEDVERSION" ]; then + echo "" + echo "NOTE: A previous upgrade to $INSTALLEDVERSION did not finish. This grid is" + echo " partially upgraded and this soup run will resume and complete it." + fi echo "" check_minimum_version @@ -1785,6 +1895,12 @@ main() { echo "Verifying Elasticsearch version compatibility across the grid before upgrading." verify_es_version_compatibility + # Pre-flight health checks: confirm the grid is in a good state before we change + # anything. These run before any modifications, so a failure exits cleanly and the + # operator can fix the issue and re-run soup. + check_cluster_health + check_fleet_server + echo "Checking for Salt Master and Minion updates." upgrade_check_salt set -e @@ -1801,6 +1917,7 @@ main() { fi if [ "$is_hotfix" == "true" ]; then + SOUP_UPGRADE_STARTED=true echo "Applying $HOTFIXVERSION hotfix" # since we don't run the backup.config_backup state on import we wont snapshot previous version states and pillars if [[ ! "$MINION_ROLE" == "import" ]]; then @@ -1811,10 +1928,16 @@ main() { create_local_directories "/opt/so/saltstack/default" apply_hotfix echo "Hotfix applied" - update_version enable_highstate highstate + # Record the hotfix only after the highstate succeeds. /etc/sohotfix is written + # solely by soup (no salt state manages it), so deferring the write means a failed + # hotfix highstate leaves the old hotfix value and re-running soup re-applies it, + # rather than reporting "already latest". The soversion/pillar writes in + # update_version are no-ops here since the version is unchanged for a hotfix. + update_version else + SOUP_UPGRADE_STARTED=true echo "" echo "Performing upgrade from Security Onion $INSTALLEDVERSION to Security Onion $NEWVERSION." echo "" @@ -1870,6 +1993,10 @@ main() { copy_new_files echo "" create_local_directories "/opt/so/saltstack/default" + # Seed the resume marker before the highstate stamps /etc/soversion to the new + # version, so an interrupted upgrade is detectable as "not finished" on re-run. + # POSTVERSION still holds the pre-upgrade (or prior resume) version here. + [ -f "$POSTVERSION_FILE" ] || echo "$POSTVERSION" > "$POSTVERSION_FILE" update_version echo "" diff --git a/salt/nginx/etc/nginx.conf b/salt/nginx/etc/nginx.conf index b7a70da2b..3f74c411c 100644 --- a/salt/nginx/etc/nginx.conf +++ b/salt/nginx/etc/nginx.conf @@ -399,7 +399,7 @@ http { error_page 429 = @error429; location @error401 { - if ($request_uri ~* (^/api/.*|^/connect/.*|^/oauth2/.*|^/.*\.map$)) { + if ($request_uri ~* (^.*/api/.*|^/connect/.*|^/oauth2/.*|^/.*\.map$)) { return 401; } diff --git a/salt/postgres/enabled.sls b/salt/postgres/enabled.sls index 20d256ae8..b0eadd205 100644 --- a/salt/postgres/enabled.sls +++ b/salt/postgres/enabled.sls @@ -19,6 +19,7 @@ include: so-postgres: docker_container.running: - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-postgres:{{ GLOBALS.so_version }} + - restart_policy: unless-stopped - hostname: so-postgres - networks: - sobridge: diff --git a/salt/registry/enabled.sls b/salt/registry/enabled.sls index 7b039b313..b2bfbc56d 100644 --- a/salt/registry/enabled.sls +++ b/salt/registry/enabled.sls @@ -17,14 +17,13 @@ include: so-dockerregistry: docker_container.running: - image: ghcr.io/security-onion-solutions/registry:3.1.1 + # Intentionally `always`-- registry is critical and must + # come back up even if it was manually stopped. + - restart_policy: always - hostname: so-registry - networks: - sobridge: - ipv4_address: {{ DOCKERMERGED.containers['so-dockerregistry'].ip }} - # Intentionally `always` (not unless-stopped) -- registry is critical infra - # and must come back up even if it was manually stopped. Do not homogenize - # to unless-stopped; see the container auto-restart section of the plan. - - restart_policy: always - port_bindings: {% for BINDING in DOCKERMERGED.containers['so-dockerregistry'].port_bindings %} - {{ BINDING }} diff --git a/salt/salt/master.sls b/salt/salt/master.sls index bb39f1395..2c47a95c6 100644 --- a/salt/salt/master.sls +++ b/salt/salt/master.sls @@ -15,7 +15,6 @@ include: - salt.minion - - salt.master.pyinotify - salt.master.boot_mine_update {% if 'vrt' in salt['pillar.get']('features', []) %} - salt.cloud diff --git a/salt/salt/master/pyinotify.sls b/salt/salt/master/pyinotify.sls deleted file mode 100644 index 8aa2f1d53..000000000 --- a/salt/salt/master/pyinotify.sls +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -pyinotify_module_package: - file.recurse: - - name: /opt/so/conf/salt/module_packages/pyinotify - - source: salt://salt/module_packages/pyinotify - - clean: True - - makedirs: True - -pyinotify_python_module_install: - cmd.run: - - name: /opt/saltstack/salt/bin/python3.10 -m pip install pyinotify --no-index --find-links=/opt/so/conf/salt/module_packages/pyinotify/ --upgrade - - onchanges: - - file: pyinotify_module_package - - failhard: True - - watch_in: - - service: salt_minion_service diff --git a/salt/salt/minion/init.sls b/salt/salt/minion/init.sls index a251aa633..1f1ec8305 100644 --- a/salt/salt/minion/init.sls +++ b/salt/salt/minion/init.sls @@ -131,11 +131,15 @@ salt_minion_service: {% endif %} - order: last -# block until the just-restarted salt-minion is back and can execute modules locally, so -# follow-on jobs and the next highstate iteration do not race the restart. onchanges + -# require on salt_minion_service catches every restart trigger uniformly because watch -# mod_watch results replace the service state's running entry. wait logic lives in -# /usr/sbin/so-salt-minion-wait (deployed by common_sbin from common/tools/sbin/). +# block until the just-restarted salt-minion daemon logs "Minion is ready to receive requests!" +# for the current instance, so follow-on jobs and the next highstate iteration do not race the +# restart. onchanges + require on salt_minion_service catches every restart trigger uniformly +# because watch mod_watch results replace the service state's running entry. wait logic lives in +# /usr/sbin/so-salt-minion-wait (deployed by salt_sbin from salt/tools/sbin/); it keys the ready +# line to the current daemon pid (resolved via systemd, not the pidfile) and corroborates with the +# master req/publish sockets. set_log_levels above enforces the log_level_logfile: info that the +# ready line depends on. salt restarts this unit with --no-block, so mod_watch returns while the old +# daemon is still up; the script waits for systemd's restart job to drain before it reads MainPID. wait_for_salt_minion_ready: cmd.run: - name: /usr/sbin/so-salt-minion-wait diff --git a/salt/salt/module_packages/pyinotify/pyinotify-0.9.6.tar.gz b/salt/salt/module_packages/pyinotify/pyinotify-0.9.6.tar.gz deleted file mode 100644 index 3150ad360..000000000 Binary files a/salt/salt/module_packages/pyinotify/pyinotify-0.9.6.tar.gz and /dev/null differ diff --git a/salt/salt/tools/sbin/so-salt-minion-wait b/salt/salt/tools/sbin/so-salt-minion-wait index a30c67e80..984f144f3 100644 --- a/salt/salt/tools/sbin/so-salt-minion-wait +++ b/salt/salt/tools/sbin/so-salt-minion-wait @@ -5,31 +5,173 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -# Block until the local salt-minion service is back up and can execute modules locally. -# Invoked from the wait_for_salt_minion_ready state in salt/minion/init.sls after -# salt_minion_service fires its watch-driven mod_watch (a non-blocking systemctl restart), -# so follow-on jobs and the next highstate iteration do not race the in-flight restart. +# Block until the just-restarted salt-minion daemon reaches the point where salt itself logs +# "Minion is ready to receive requests!". Invoked from the wait_for_salt_minion_ready state in +# salt/minion/init.sls after salt_minion_service fires its watch-driven restart, so follow-on jobs +# and the next highstate iteration do not race it. +# +# Salt logs that line from Minion.tune_in() only after sync_connect_master() returns, which means +# the pub channel authenticated, the long-running req channel connected, and _post_master_init() +# finished loading modules and compiling pillar. Two signals reproduce that: +# +# 1. Primary the pid-tagged ready line in the minion log. Salt's log_fmt_logfile embeds +# [%(process)d] just before the message, so this is keyed to one daemon instance. +# 2. Corroborating that same pid holds an ESTABLISHED req connection to a master on 4506 plus a +# second (publish) connection to that same master IP on another port. The publish +# port is learned from the master's auth reply and is absent from minion config, +# so it is derived from the connection rather than read from config. +# +# The daemon pid is resolved from systemd, never from /var/run/salt-minion.pid. salt_minion() runs +# the real minion in a multiprocessing child; that child writes the pidfile, owns the sockets and +# logs the ready line, while systemd's MainPID is the parent. During a restart the pidfile can still +# name the OLD child, whose own ready line is already in the log -- matching it would report ready +# instantly. Children of the current MainPID structurally exclude the old instance. +# +# That is only true once systemd has actually swapped MainPID. Salt restarts this unit with +# --no-block (salt/modules/systemd_service.py:_no_block_default returns True for salt-minion), so +# service.restart returns as soon as the job is enqueued and the state proceeds to run this script +# while the OLD daemon is still up -- observed at ~7s before MainPID flips. Reading MainPID in that +# window names the outgoing instance, which is still fully connected and has its own ready line, so +# every gate below would pass on the daemon that is about to die. Wait for systemd's job queue for +# the unit to drain first; that is the deterministic "the swap has happened" signal. . /usr/sbin/so-common -# Initial sleep gives the systemctl restart (--no-block by default for salt-minion on -# >=3006.15) time to begin tearing down the old process before we probe for readiness. +set -u + INITIAL_SLEEP=3 TIMEOUT=120 -PING_TIMEOUT=5 +MASTER_PORT=4506 +LOG_TAIL_LINES=10000 +DEFAULT_LOG_FILE="/opt/so/log/salt/minion" +LOG_FILE="$DEFAULT_LOG_FILE" + +# Decide whether the ready line can ever appear. salt-call --local sets file_client=local, so this +# reads the merged config (honoring minion.d overrides) without contacting the master. salt defaults +# log_level_logfile to None, meaning it inherits log_level, so resolve that before deciding. +LOG_LEVEL_LOGFILE=$(salt-call --local --out=newline_values_only config.get log_level_logfile 2>/dev/null | head -n1) +case "${LOG_LEVEL_LOGFILE,,}" in + ""|none) LOG_LEVEL_LOGFILE=$(salt-call --local --out=newline_values_only config.get log_level 2>/dev/null | head -n1) ;; +esac + +case "${LOG_LEVEL_LOGFILE,,}" in + all|garbage|trace|debug|profile|info) USE_LOG_GATE=1 ;; + *) USE_LOG_GATE=0 ;; +esac + +if [ "$USE_LOG_GATE" -eq 1 ]; then + LOG_FILE=$(salt-call --local --out=newline_values_only config.get log_file 2>/dev/null | head -n1) + [ -z "$LOG_FILE" ] && LOG_FILE="$DEFAULT_LOG_FILE" + [ -d "$(dirname "$LOG_FILE")" ] || USE_LOG_GATE=0 +fi + +if command -v ss >/dev/null 2>&1; then + USE_SOCKET_GATE=1 +else + USE_SOCKET_GATE=0 +fi + +if [ "$USE_LOG_GATE" -eq 0 ] && [ "$USE_SOCKET_GATE" -eq 0 ]; then + echo "so-salt-minion-wait: no usable readiness signal (log_level_logfile='${LOG_LEVEL_LOGFILE:-unset}', ss not found)" >&2 + exit 1 +fi + +if [ "$USE_LOG_GATE" -eq 1 ] && [ "$USE_SOCKET_GATE" -eq 1 ]; then + echo "so-salt-minion-wait: gating on pid-tagged ready line in ${LOG_FILE} plus master sockets" +elif [ "$USE_LOG_GATE" -eq 1 ]; then + echo "so-salt-minion-wait: ss not found; gating on pid-tagged ready line in ${LOG_FILE} only" +else + echo "so-salt-minion-wait: INFO file logging unavailable (log_level_logfile='${LOG_LEVEL_LOGFILE:-unset}'); gating on master sockets only" +fi + +# True while systemd still has a queued or running job for the unit, i.e. an in-flight --no-block +# restart. MainPID still names the outgoing daemon until this drains. The unit name is passed as a +# filter and grepped as well, so this stays correct if an older systemctl ignores the filter. +restart_pending() { + systemctl list-jobs --no-legend salt-minion.service 2>/dev/null | grep -q 'salt-minion\.service' +} + +# Emit the pid(s) of the current daemon instance. systemd's MainPID is the parent keepalive process; +# its child runs tune_in. Fall back to MainPID when there is no child (--disable-keepalive path). +resolve_daemon_pids() { + local mainpid children + mainpid=$(systemctl show -p MainPID --value salt-minion 2>/dev/null) + if [ -z "$mainpid" ] || [ "$mainpid" = "0" ]; then + return 1 + fi + children=$(pgrep -P "$mainpid" 2>/dev/null) + printf '%s\n' "${children:-$mainpid}" +} + +# True iff the ready line tagged with this pid is in the current or most recently rotated log. +ready_logged() { + local pid=$1 f + for f in "$LOG_FILE" "$LOG_FILE.1"; do + [ -r "$f" ] || continue + if tail -n "$LOG_TAIL_LINES" "$f" 2>/dev/null | grep -Fq "[$pid] Minion is ready to receive requests!"; then + return 0 + fi + done + return 1 +} + +# True iff this pid holds an ESTABLISHED req connection to a master on MASTER_PORT and a second +# ESTABLISHED connection to that same master IP on another port. The trailing comma in "pid=N," +# keeps pid=123 from matching pid=1234. Grid comms are IPv4 (the unit's ExecStartPre gates on ip -4). +socket_ready() { + local pid=$1 mip master_ips + master_ips=$(ss -tnp state established "dport = :${MASTER_PORT}" 2>/dev/null \ + | grep -F "pid=${pid}," \ + | grep -oE "[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+:${MASTER_PORT}" \ + | sed "s/:${MASTER_PORT}\$//" \ + | sort -u) + [ -z "$master_ips" ] && return 1 + for mip in $master_ips; do + if ss -tnp state established "dst ${mip} and dport != :${MASTER_PORT}" 2>/dev/null | grep -qF "pid=${pid},"; then + return 0 + fi + done + return 1 +} + +instance_ready() { + local pid=$1 + if [ "$USE_LOG_GATE" -eq 1 ] && ! ready_logged "$pid"; then + return 1 + fi + if [ "$USE_SOCKET_GATE" -eq 1 ] && ! socket_ready "$pid"; then + return 1 + fi + return 0 +} sleep "$INITIAL_SLEEP" elapsed="$INITIAL_SLEEP" +pids="" +announced_pending=0 while [ "$elapsed" -lt "$TIMEOUT" ]; do - if systemctl is-active --quiet salt-minion \ - && salt-call --local --timeout="$PING_TIMEOUT" --out=quiet test.ping >/dev/null 2>&1; then - echo "salt-minion ready after ${elapsed}s" - exit 0 + if restart_pending; then + # An in-flight --no-block restart: MainPID still names the outgoing daemon. Evaluating now + # would bless the instance that is about to be torn down. + if [ "$announced_pending" -eq 0 ]; then + echo "so-salt-minion-wait: systemd restart job in flight; waiting for it to drain" + announced_pending=1 + fi + elif pids=$(resolve_daemon_pids); then + # shellcheck disable=SC2086 + for pid in $pids; do + if instance_ready "$pid"; then + echo "salt-minion (pid ${pid}) ready after ${elapsed}s" + exit 0 + fi + done fi sleep 1 elapsed=$((elapsed + 1)) done -echo "salt-minion did not become ready within ${TIMEOUT}s" >&2 +mainpid=$(systemctl show -p MainPID --value salt-minion 2>/dev/null) +restart_pending && pending=yes || pending=no +echo "salt-minion did not become ready within ${TIMEOUT}s (MainPID=${mainpid:-unknown}, candidate pids='${pids:-none}', restart_job_pending=${pending}, log_gate=${USE_LOG_GATE}, socket_gate=${USE_SOCKET_GATE}, log_file=${LOG_FILE})" >&2 exit 1 diff --git a/salt/soc/defaults.map.jinja b/salt/soc/defaults.map.jinja index 2821bb8e5..93fb63efe 100644 --- a/salt/soc/defaults.map.jinja +++ b/salt/soc/defaults.map.jinja @@ -8,6 +8,7 @@ {% from 'docker/docker.map.jinja' import DOCKERMERGED -%} {% set INFLUXDB_TOKEN = salt['pillar.get']('influxdb:token') %} {% import_text 'influxdb/metrics_link.txt' as METRICS_LINK %} +{% from 'telegraf/map.jinja' import TELEGRAFMERGED %} {% for module, application_url in GLOBALS.application_urls.items() %} {% do SOCDEFAULTS.soc.config.server.modules[module].update({'hostUrl': application_url}) %} @@ -24,13 +25,22 @@ {% do SOCDEFAULTS.soc.config.server.modules.elastic.update({'username': GLOBALS.elasticsearch.auth.users.so_elastic_user.user, 'password': GLOBALS.elasticsearch.auth.users.so_elastic_user.pass}) %} -{% do SOCDEFAULTS.soc.config.server.modules.influxdb.update({'hostUrl': 'https://' ~ GLOBALS.influxdb_host ~ ':8086'}) %} -{% do SOCDEFAULTS.soc.config.server.modules.influxdb.update({'token': INFLUXDB_TOKEN}) %} -{% for tool in SOCDEFAULTS.soc.config.server.client.tools %} -{% if tool.name == "toolInfluxDb" and METRICS_LINK | length > 0 %} -{% do tool.update({'link': METRICS_LINK}) %} -{% endif %} -{% endfor %} +{% if TELEGRAFMERGED.output == 'POSTGRES' %} +{% for tool in SOCDEFAULTS.soc.config.server.client.tools %} +{% if tool.name == "toolInfluxDb" %} +{% do SOCDEFAULTS.soc.config.server.client.tools.remove(tool) %} +{% endif %} +{% endfor %} + +{% else %} +{% do SOCDEFAULTS.soc.config.server.modules.influxdb.update({'hostUrl': 'https://' ~ GLOBALS.influxdb_host ~ ':8086'}) %} +{% do SOCDEFAULTS.soc.config.server.modules.influxdb.update({'token': INFLUXDB_TOKEN}) %} +{% for tool in SOCDEFAULTS.soc.config.server.client.tools %} +{% if tool.name == "toolInfluxDb" and METRICS_LINK | length > 0 %} +{% do tool.update({'link': METRICS_LINK}) %} +{% endif %} +{% endfor %} +{% endif %} {% do SOCDEFAULTS.soc.config.server.modules.statickeyauth.update({'anonymousCidr': DOCKERMERGED.range, 'apiKey': pillar.sensoroni.config.sensoronikey}) %} diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index e95008a96..8df30736f 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1515,6 +1515,8 @@ soc: assistant: systemPromptAddendum: "" systemPromptAddendumMaxLength: 50000 + maxSubSessionTokens: 0 + maxDelegationDepth: 5 adapters: - name: SOAI protocol: securityonion_ai_cloud @@ -1526,6 +1528,10 @@ soc: serviceAccountJSON: "" serviceAccountLocation: "" healthTimeoutSeconds: 5 + agentic: false + agentMapping: + Orchestrator: sonnet + Hunter: sonnet onionconfig: saltstackDir: /opt/so/saltstack bypassEnabled: false @@ -2695,6 +2701,8 @@ soc: thresholdColorRatioLow: 0.5 thresholdColorRatioMed: 0.75 thresholdColorRatioMax: 1 + toolBusyMaxRetries: 30 + toolBusyRetryDelayMs: 1000 availableModels: - id: sonnet displayName: Claude Sonnet diff --git a/salt/soc/merged.map.jinja b/salt/soc/merged.map.jinja index cfc0fafbd..a421711e2 100644 --- a/salt/soc/merged.map.jinja +++ b/salt/soc/merged.map.jinja @@ -7,6 +7,11 @@ {% from 'soc/defaults.map.jinja' import SOCDEFAULTS with context %} {% from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_NODES %} {% from 'manager/map.jinja' import MANAGERMERGED %} +{% from 'telegraf/map.jinja' import TELEGRAFMERGED %} +{%- set PG_ENTRY = salt['pillar.get']('telegraf:postgres_creds:' ~ grains.id, {}) %} +{%- set PG_USER = PG_ENTRY.get('user', '') %} +{%- set PG_PASS = PG_ENTRY.get('pass', '') %} + {% set DOCKER_EXTRA_HOSTS = ELASTICSEARCH_NODES %} {% do DOCKER_EXTRA_HOSTS.append({GLOBALS.influxdb_host:pillar.node_data[GLOBALS.influxdb_host].ip}) %} @@ -75,6 +80,20 @@ {% do SOCMERGED.config.server.update({'airgapEnabled': false}) %} {% endif %} +{# Define the postgresmetrics module if telegraf is setup to only use Postgres #} +{% if TELEGRAFMERGED.output != 'INFLUXDB' and PG_USER and PG_PASS %} +{% do SOCMERGED.config.server.modules.update({ + 'postgresmetrics': { + 'database': 'so_telegraf', + 'host': GLOBALS.manager_ip, + 'password': PG_PASS, + 'port': 5432, + 'sslMode': 'allow', + 'user': PG_USER, + } + }) %} +{% do SOCMERGED.config.server.modules.pop('influxdb') %} +{% endif %} {# Define the Detections custom ruleset that should always be present #} {% set CUSTOM_RULESET = { diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index 594dbf88c..47c7ea3ee 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -727,6 +727,16 @@ soc: description: Maximum length of the system prompt addendum. Longer prompts will be truncated. global: True advanced: True + maxSubSessionTokens: + description: Maximum number of output tokens a delegated sub-session may generate across all of its turns. When the budget is reached, the sub-agent is halted and its result is returned to the parent agent. Set to 0 to disable the limit. + global: True + advanced: True + forcedType: int + maxDelegationDepth: + description: Maximum delegation nesting depth for sub-agents. For example, a value of 2 lets the main agent delegate to a sub-agent that may itself delegate one level deeper. Any deeper delegation is refused and the requesting agent continues without it. Set to 0 to disable the limit. + global: True + advanced: True + forcedType: int adapters: description: Configuration for AI adapters used by the Onion AI assistant. Please see documentation for help on which fields are required for which protocols. global: True @@ -765,12 +775,29 @@ soc: label: Health Timeout Seconds required: False forcedType: int + agentic: + description: Indicates if the Assistant Module should operate in agentic mode or not. If true, agents can work together to solve tasks. + global: True + forcedType: bool + agentMapping: + Orchestrator: + description: The initial agent in most agentic conversations. This agent will delegate requests to specialized agents. + global: True + Hunter: + description: This agent is specialized in querying events. + global: True client: assistant: enabled: description: Set to true to enable the Onion AI assistant in SOC. global: True forcedType: bool + toolBusyMaxRetries: + description: How many times to retry auto approving a tool while a tool is already running. + global: True + toolBusyRetryDelayMs: + description: How long in milliseconds to wait between each retry when auto approving a tool. + global: True investigationPrompt: description: Prompt given to Onion AI when beginning an investigation. global: True diff --git a/salt/strelka/backend/enabled.sls b/salt/strelka/backend/enabled.sls index 8c71bdf68..e62349f70 100644 --- a/salt/strelka/backend/enabled.sls +++ b/salt/strelka/backend/enabled.sls @@ -15,6 +15,7 @@ include: strelka_backend: docker_container.running: - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-strelka-backend:{{ GLOBALS.so_version }} + - restart_policy: unless-stopped - binds: - /opt/so/conf/strelka/backend/:/etc/strelka/:ro - /opt/so/conf/strelka/rules/compiled/:/etc/yara/:ro @@ -47,11 +48,6 @@ strelka_backend: - {{ ULIMIT.name }}={{ ULIMIT.soft }}:{{ ULIMIT.hard }} {% endfor %} {% endif %} - # Intentionally `on-failure` (not unless-stopped) -- strelka backend shuts - # down cleanly during rule reloads and we do not want those clean exits to - # trigger an auto-restart. Do not homogenize; see the container - # auto-restart section of the plan. - - restart_policy: on-failure - watch: - file: strelkasensorcompiledrules - file: backend_backend_config diff --git a/salt/suricata/enabled.sls b/salt/suricata/enabled.sls index 53f367971..d9206798e 100644 --- a/salt/suricata/enabled.sls +++ b/salt/suricata/enabled.sls @@ -17,8 +17,8 @@ include: so-suricata: docker_container.running: - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-suricata:{{ GLOBALS.so_version }} - - privileged: True - restart_policy: unless-stopped + - privileged: True - environment: - INTERFACE={{ GLOBALS.sensor.interface }} {% if DOCKERMERGED.containers['so-suricata'].extra_env %} diff --git a/salt/telegraf/soc_telegraf.yaml b/salt/telegraf/soc_telegraf.yaml index 4b9a2e3d1..0decdd06a 100644 --- a/salt/telegraf/soc_telegraf.yaml +++ b/salt/telegraf/soc_telegraf.yaml @@ -5,7 +5,7 @@ telegraf: advanced: True helpLink: influxdb output: - description: Selects the backend(s) Telegraf writes metrics to. INFLUXDB keeps the current behavior; POSTGRES writes to the grid's Postgres instance; BOTH dual-writes for migration validation. + description: Selects the backend(s) Telegraf writes metrics to. INFLUXDB keeps the current behavior; POSTGRES writes to the grid's Postgres instance; BOTH dual-writes for migration validation. When set to BOTH, the grid screen's metrics are pulled from Postgres, and the InfluxDB tool link remains visible. When set to POSTGRES, the InfluxDB tool link is removed. options: - INFLUXDB - POSTGRES diff --git a/salt/zeek/enabled.sls b/salt/zeek/enabled.sls index 355e555b3..ec01693d7 100644 --- a/salt/zeek/enabled.sls +++ b/salt/zeek/enabled.sls @@ -16,9 +16,9 @@ include: so-zeek: docker_container.running: - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-zeek:{{ GLOBALS.so_version }} + - restart_policy: unless-stopped - start: True - privileged: True - - restart_policy: unless-stopped {% if DOCKERMERGED.containers['so-zeek'].ulimits %} - ulimits: {% for ULIMIT in DOCKERMERGED.containers['so-zeek'].ulimits %} diff --git a/setup/so-setup b/setup/so-setup index a44934088..896505ba5 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -793,7 +793,7 @@ if ! [[ -f $install_opt_file ]]; then logCmd "so-soc-restart" title "Setting up Elastic Fleet" logCmd "salt-call state.apply elasticfleet.config" - if ! logCmd so-elastic-fleet-setup; then + if ! so-elastic-fleet-setup; then fail_setup "Failed to run so-elastic-fleet-setup" fi mark_setup_complete