From 1fed3cf4746fc5c2c63077243cdd3bc9d457b774 Mon Sep 17 00:00:00 2001 From: weslambert Date: Tue, 17 Jan 2023 16:54:31 -0500 Subject: [PATCH 01/19] Disable Kibana's native base path rewrite and add publicBaseUrl --- salt/kibana/defaults.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/kibana/defaults.yaml b/salt/kibana/defaults.yaml index 549b8549b..d854b3e17 100644 --- a/salt/kibana/defaults.yaml +++ b/salt/kibana/defaults.yaml @@ -5,6 +5,8 @@ kibana: name: kibana host: "0.0.0.0" basePath: /kibana + publicBaseUrl: https://{{salt['pillar.get']('global:url_base')}}/kibana + rewriteBasePath: false elasticsearch: ssl: verificationMode: none From 7a499c9051c1cf141d97f0a1f1b91fc377c7aa69 Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 18 Jan 2023 17:24:07 -0500 Subject: [PATCH 02/19] Modify default 'logs-*' template priority --- salt/elasticsearch/defaults.yaml | 61 ++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index ff96d139b..f883503a0 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -57,6 +57,67 @@ elasticsearch: elasticsearch: deprecation: ERROR index_settings: + so-logs: + index_sorting: False + index_template: + index_patterns: + - "logs-*" + data_stream: + hidden: false + allow_custom_routing: false + template: + settings: + index: + number_of_replicas: 0 + lifecycle: + name: logs + codec: best_compression + routing: + allocation: + include: + _tier_preferences: data_hot + query: + default_field: + - message + mappings: + dynamic_templates: + - match_ip: + match: ip + match_mapping_type: string + mapping: + allow_custom_routing: false + template: + settings: + index: + number_of_replicas: 0 + lifecycle: + name: logs + codec: best_compression + routing: + allocation: + include: + _tier_preferences: data_hot + query: + default_field: + - message + mappings: + dynamic_templates: + - match_ip: + match: ip + match_mapping_type: string + mapping: + type: ip + - match_message: + match: message + match_mapping_type: string + mapping: + type: match_only_text + - strings_as_keyword: + match_mapping_type: string + mapping: + ignore_above: 1024 + type: keyword + priority: 125 so-logs-elastic_agent.apm_server: index_sorting: False index_template: From e83e54936ed410176a4f587147ccc14c01f65814 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Thu, 19 Jan 2023 07:25:25 -0500 Subject: [PATCH 03/19] Temp disable Elastic Registry Repo --- salt/allowed_states.map.jinja | 4 ---- salt/common/tools/sbin/so-image-common | 1 - salt/kibana/defaults.yaml | 1 - salt/top.sls | 1 - 4 files changed, 7 deletions(-) diff --git a/salt/allowed_states.map.jinja b/salt/allowed_states.map.jinja index 192a52dd3..2f6cc60a0 100644 --- a/salt/allowed_states.map.jinja +++ b/salt/allowed_states.map.jinja @@ -37,7 +37,6 @@ 'soc', 'kratos', 'elastic-fleet', - 'elastic-fleet-package-registry', 'firewall', 'idstools', 'suricata.manager', @@ -123,7 +122,6 @@ 'soc', 'kratos', 'elastic-fleet', - 'elastic-fleet-package-registry', 'firewall', 'idstools', 'suricata.manager', @@ -144,7 +142,6 @@ 'soc', 'kratos', 'elastic-fleet', - 'elastic-fleet-package-registry', 'firewall', 'manager', 'idstools', @@ -175,7 +172,6 @@ 'soc', 'kratos', 'elastic-fleet', - 'elastic-fleet-package-registry', 'firewall', 'idstools', 'suricata.manager', diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common index 9ba66ced8..3851d8b4a 100755 --- a/salt/common/tools/sbin/so-image-common +++ b/salt/common/tools/sbin/so-image-common @@ -44,7 +44,6 @@ container_list() { "so-elastalert" "so-elastic-agent" "so-elastic-agent-builder" - "so-elastic-fleet-package-registry" "so-elasticsearch" "so-filebeat" "so-grafana" diff --git a/salt/kibana/defaults.yaml b/salt/kibana/defaults.yaml index d854b3e17..159465508 100644 --- a/salt/kibana/defaults.yaml +++ b/salt/kibana/defaults.yaml @@ -33,7 +33,6 @@ kibana: kibanaServer: hostname: localhost fleet: - registryUrl: "" packages: - name: fleet_server version: latest diff --git a/salt/top.sls b/salt/top.sls index ada947ec8..e29d3b081 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -191,7 +191,6 @@ base: {%- if REDIS %} - redis {%- endif %} - - elastic-fleet-package-registry {%- if KIBANA %} - kibana.so_savedobjects_defaults {%- endif %} From 05a6d702b07b6bf1b6ae3a9f3a74a3f04586318d Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 19 Jan 2023 10:03:03 -0500 Subject: [PATCH 04/19] Add logic to determine if setup succeeded and provide relevant output --- salt/common/tools/sbin/so-common | 20 +----- setup/so-setup | 4 +- setup/so-verify | 108 +++++++++++++++++++++++++++++++ 3 files changed, 113 insertions(+), 19 deletions(-) create mode 100644 setup/so-verify diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 43e6f4446..c4a3fdbaf 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -225,31 +225,17 @@ init_monitor() { } is_manager_node() { - # Check to see if this is a manager node - role=$(lookup_role) - is_single_node_grid && return 0 - [ $role == 'manager' ] && return 0 - [ $role == 'managersearch' ] && return 0 - [ $role == 'helix' ] && return 0 - return 1 + grep "role: so-" /etc/salt/grains | grep -E "manager|eval|managersearch|standalone|import" &> /dev/null } is_sensor_node() { # Check to see if this is a sensor (forward) node - role=$(lookup_role) is_single_node_grid && return 0 - [ $role == 'sensor' ] && return 0 - [ $role == 'heavynode' ] && return 0 - [ $role == 'helix' ] && return 0 - return 1 + grep "role: so-" /etc/salt/grains | grep -E "sensor|heavynode|helix" &> /dev/null } is_single_node_grid() { - role=$(lookup_role) - [ $role == 'eval' ] && return 0 - [ $role == 'standalone' ] && return 0 - [ $role == 'import' ] && return 0 - return 1 + grep "role: so-" /etc/salt/grains | grep -E "eval|standalone|import" &> /dev/null } lookup_bond_interfaces() { diff --git a/setup/so-setup b/setup/so-setup index 99e218a61..80c6372c9 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -585,7 +585,7 @@ if ! [[ -f $install_opt_file ]]; then fi checkin_at_boot set_initial_firewall_access - whiptail_setup_complete + verify_setup $setup_type else touch /root/accept_changes mkdir -p /opt/so @@ -608,7 +608,7 @@ if ! [[ -f $install_opt_file ]]; then configure_minion "$minion_type" drop_install_options checkin_at_boot - whiptail_setup_complete + verify_setup $setup_type fi # Need to make sure the latest install is located on the web server of the manager to check the versions and donwload the code if required diff --git a/setup/so-verify b/setup/so-verify new file mode 100644 index 000000000..22947df7e --- /dev/null +++ b/setup/so-verify @@ -0,0 +1,108 @@ +#!/bin/bash + +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +cd "$(dirname "$0")" || exit 255 + +source ../salt/common/tools/sbin/so-common +source ./so-functions +source ./so-whiptail +source ./so-variables + +setup_type=$1 + +setup_in_progress() { + ps -ef | grep so-setup | grep -v grep &> /dev/null +} + +using_iso() { + if [ "$setup_type" == "iso" ]; then + return 0 + fi + return 0 +} + +whipit() { + if [[ $exit_code -eq 0 ]]; then + whiptail_setup_complete + else + whiptail_setup_failed + fi +} + +# Check entire setup log for errors or unexpected salt states +log_has_errors() { + grep -E "FAILED|Failed|failed|ERROR|Error|Result: False" "$setup_log" | \ + grep -vE "[ERROR ] The Salt Master has cached the public key for this node" | \ + grep -vE "Minion failed to authenticate with master" | \ + grep -vE "perl-Error-" | \ + grep -vE "Failed:\s*?[0-9]+" | \ + grep -vE "Status .* was not found" | \ + grep -vE "Uncaught exception, closing connection" | \ + grep -vE "Exception in callback None" | \ + grep -vE "deprecation: ERROR" | \ + grep -vE "code: 100" | \ + grep -vE "Running scope as unit" &> "$error_log" + + if [[ $? -eq 0 ]]; then + return 0 + fi + return 1 +} + +# For ISO installs, we know nothing else can be running on this server, so there should be +# nothing in any mail spool dir. +cron_error_in_mail_spool() { + if find /var/spool/mail/ -type f -size +0 &> /dev/null; then + return 0 + fi + return 1 +} + +# so-setup must return a 0 exit code, indicating all containers are up and healthy. Will retry for a limited +# time before giving up. +status_failed() { + max_retries=120 + wait_secs=10 + retry_attempts=0 + while ! so-status -q; do + if [[ $retry_attempts -eq $max_retries ]]; then + return 0 + fi + retry_attempts=$((retry_attempts+1)) + echo "INFO: so-status returned non-zero exit code; will retry in $wait_secs seconds ($retry_attempts/$max_retries)" + sleep $wait_secs + done + return 1 +} + +main() { + exit_code=0 + if log_has_errors; then + echo "WARNING: Errors detected during setup" + exit_code=1 + fi + + if using_iso && cron_error_in_mail_spool; then + echo "WARNING: Unexpected cron job output in mail spool" + exit_code=1 + fi + + if is_manager_node && status_failed; then + echo "WARNING: Containers are not in a healthy state" + exit_code=1 + fi + + if [[ exit_code -eq 0 ]]; then + echo "Successfully completed setup!" + fi + + setup_in_progress && whipit $exit_code + + exit $exit_code +} + +main From 7d3f6121ebb245af1267af1809fd2db3d03aab0d Mon Sep 17 00:00:00 2001 From: weslambert Date: Thu, 19 Jan 2023 10:29:10 -0500 Subject: [PATCH 05/19] Remove default "logs-*" template settings for now --- salt/elasticsearch/defaults.yaml | 61 -------------------------------- 1 file changed, 61 deletions(-) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index f883503a0..ff96d139b 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -57,67 +57,6 @@ elasticsearch: elasticsearch: deprecation: ERROR index_settings: - so-logs: - index_sorting: False - index_template: - index_patterns: - - "logs-*" - data_stream: - hidden: false - allow_custom_routing: false - template: - settings: - index: - number_of_replicas: 0 - lifecycle: - name: logs - codec: best_compression - routing: - allocation: - include: - _tier_preferences: data_hot - query: - default_field: - - message - mappings: - dynamic_templates: - - match_ip: - match: ip - match_mapping_type: string - mapping: - allow_custom_routing: false - template: - settings: - index: - number_of_replicas: 0 - lifecycle: - name: logs - codec: best_compression - routing: - allocation: - include: - _tier_preferences: data_hot - query: - default_field: - - message - mappings: - dynamic_templates: - - match_ip: - match: ip - match_mapping_type: string - mapping: - type: ip - - match_message: - match: message - match_mapping_type: string - mapping: - type: match_only_text - - strings_as_keyword: - match_mapping_type: string - mapping: - ignore_above: 1024 - type: keyword - priority: 125 so-logs-elastic_agent.apm_server: index_sorting: False index_template: From c5260e4787cd83153dbf5dc0be774e5f1e4706e1 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 19 Jan 2023 11:25:59 -0500 Subject: [PATCH 06/19] verify setup --- setup/so-setup | 4 ++-- setup/so-verify | 12 +++--------- 2 files changed, 5 insertions(+), 11 deletions(-) mode change 100644 => 100755 setup/so-verify diff --git a/setup/so-setup b/setup/so-setup index 80c6372c9..ad4f57d55 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -585,7 +585,7 @@ if ! [[ -f $install_opt_file ]]; then fi checkin_at_boot set_initial_firewall_access - verify_setup $setup_type + so-verify $setup_type else touch /root/accept_changes mkdir -p /opt/so @@ -608,7 +608,7 @@ if ! [[ -f $install_opt_file ]]; then configure_minion "$minion_type" drop_install_options checkin_at_boot - verify_setup $setup_type + so-verify $setup_type fi # Need to make sure the latest install is located on the web server of the manager to check the versions and donwload the code if required diff --git a/setup/so-verify b/setup/so-verify old mode 100644 new mode 100755 index 22947df7e..26581e673 --- a/setup/so-verify +++ b/setup/so-verify @@ -84,19 +84,13 @@ main() { if log_has_errors; then echo "WARNING: Errors detected during setup" exit_code=1 - fi - - if using_iso && cron_error_in_mail_spool; then + elif using_iso && cron_error_in_mail_spool; then echo "WARNING: Unexpected cron job output in mail spool" exit_code=1 - fi - - if is_manager_node && status_failed; then + elif is_manager_node && status_failed; then echo "WARNING: Containers are not in a healthy state" exit_code=1 - fi - - if [[ exit_code -eq 0 ]]; then + else echo "Successfully completed setup!" fi From 4369d2385b2abcc1e336cc45b92e764083dfb57e Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Thu, 19 Jan 2023 11:45:13 -0500 Subject: [PATCH 07/19] Temp disable Elastic Registry Repo --- salt/kibana/config.map.jinja | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/kibana/config.map.jinja b/salt/kibana/config.map.jinja index 98ca7338e..9e75f1726 100644 --- a/salt/kibana/config.map.jinja +++ b/salt/kibana/config.map.jinja @@ -7,7 +7,6 @@ {% do KIBANACONFIG.kibana.config.elasticsearch.update({'username': salt['pillar.get']('elasticsearch:auth:users:so_kibana_user:user'), 'password': salt['pillar.get']('elasticsearch:auth:users:so_kibana_user:pass')}) %} -{% do KIBANACONFIG.kibana.config.xpack.fleet.update({'registryUrl': 'http://' ~ GLOBALS.manager_ip ~ ':8080'}) %} {% if salt['pillar.get']('kibana:secrets') %} {% do KIBANACONFIG.kibana.config.xpack.update({'encryptedSavedObjects': {'encryptionKey': pillar['kibana']['secrets']['encryptedSavedObjects']['encryptionKey']}}) %} From 6b7a8e1fcd3db82d78a5625d24096cad5c36d17a Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 19 Jan 2023 12:53:24 -0500 Subject: [PATCH 08/19] fix verify path --- setup/so-setup | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index ad4f57d55..fb19a9120 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -585,7 +585,7 @@ if ! [[ -f $install_opt_file ]]; then fi checkin_at_boot set_initial_firewall_access - so-verify $setup_type + ./so-verify $setup_type else touch /root/accept_changes mkdir -p /opt/so @@ -608,7 +608,7 @@ if ! [[ -f $install_opt_file ]]; then configure_minion "$minion_type" drop_install_options checkin_at_boot - so-verify $setup_type + ./so-verify $setup_type fi # Need to make sure the latest install is located on the web server of the manager to check the versions and donwload the code if required From 59177288efb054cb1f4c2daba95bb160284a9fd3 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 19 Jan 2023 13:56:14 -0500 Subject: [PATCH 09/19] correct grep patterns --- setup/so-verify | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/so-verify b/setup/so-verify index 26581e673..c4af1d424 100755 --- a/setup/so-verify +++ b/setup/so-verify @@ -36,8 +36,8 @@ whipit() { # Check entire setup log for errors or unexpected salt states log_has_errors() { grep -E "FAILED|Failed|failed|ERROR|Error|Result: False" "$setup_log" | \ - grep -vE "[ERROR ] The Salt Master has cached the public key for this node" | \ - grep -vE "Minion failed to authenticate with master" | \ + grep -vE "The Salt Master has cached the public key for this node" | \ + grep -vE "Minion failed to authenticate with the master" | \ grep -vE "perl-Error-" | \ grep -vE "Failed:\s*?[0-9]+" | \ grep -vE "Status .* was not found" | \ From 79fb5dc52589bb469949c75b8265820600fd33f9 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 19 Jan 2023 14:19:55 -0500 Subject: [PATCH 10/19] prevent false success occurring when deleting the grafana dashboard --- .../tools/sbin/so-grafana-dashboard-folder-delete | 3 +++ setup/so-verify | 11 +++++++++++ 2 files changed, 14 insertions(+) diff --git a/salt/common/tools/sbin/so-grafana-dashboard-folder-delete b/salt/common/tools/sbin/so-grafana-dashboard-folder-delete index f3338de84..0783fccea 100755 --- a/salt/common/tools/sbin/so-grafana-dashboard-folder-delete +++ b/salt/common/tools/sbin/so-grafana-dashboard-folder-delete @@ -1,5 +1,8 @@ # this script is used to delete the default Grafana dashboard folders that existed prior to Grafana dashboard and Salt management changes in 2.3.70 +# Exit if an error occurs. The next highstate will retry. +set -e + folders=$(curl -X GET http://admin:{{salt['pillar.get']('secrets:grafana_admin')}}@localhost:3000/api/folders | jq -r '.[] | @base64') delfolder=("Manager" "Manager Search" "Sensor Nodes" "Search Nodes" "Standalone" "Eval Mode") diff --git a/setup/so-verify b/setup/so-verify index c4af1d424..234ef2dec 100755 --- a/setup/so-verify +++ b/setup/so-verify @@ -35,9 +35,20 @@ whipit() { # Check entire setup log for errors or unexpected salt states log_has_errors() { + # Ignore salt mast cached public key and minion failed to auth because this is a test + # to see if the salt key had already been accepted. + + # Ignore failed to connect to ::1 since we have most curls wrapped in a retry. + + # Ignore perl-Error- since that is the name of a Perl package SO installs. + + # Ignore Failed: 0 since that is the salt state output, and we detect state failures + # via Result: False already. + grep -E "FAILED|Failed|failed|ERROR|Error|Result: False" "$setup_log" | \ grep -vE "The Salt Master has cached the public key for this node" | \ grep -vE "Minion failed to authenticate with the master" | \ + grep -vE "Failed to connect to ::1" | \ grep -vE "perl-Error-" | \ grep -vE "Failed:\s*?[0-9]+" | \ grep -vE "Status .* was not found" | \ From 739c174898d95f92f8647da772274a7aa9061c89 Mon Sep 17 00:00:00 2001 From: Wes Date: Thu, 19 Jan 2023 21:50:10 +0000 Subject: [PATCH 11/19] Add scripts for starting, stopping, and restarting Sensoroni --- salt/common/tools/sbin/so-sensoroni-restart | 12 ++++++++++++ salt/common/tools/sbin/so-sensoroni-start | 12 ++++++++++++ salt/common/tools/sbin/so-sensoroni-stop | 12 ++++++++++++ 3 files changed, 36 insertions(+) create mode 100755 salt/common/tools/sbin/so-sensoroni-restart create mode 100755 salt/common/tools/sbin/so-sensoroni-start create mode 100755 salt/common/tools/sbin/so-sensoroni-stop diff --git a/salt/common/tools/sbin/so-sensoroni-restart b/salt/common/tools/sbin/so-sensoroni-restart new file mode 100755 index 000000000..f17de2b58 --- /dev/null +++ b/salt/common/tools/sbin/so-sensoroni-restart @@ -0,0 +1,12 @@ +#!/bin/bash + +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + + + +. /usr/sbin/so-common + +/usr/sbin/so-restart sensoroni $1 diff --git a/salt/common/tools/sbin/so-sensoroni-start b/salt/common/tools/sbin/so-sensoroni-start new file mode 100755 index 000000000..96ebc3fb6 --- /dev/null +++ b/salt/common/tools/sbin/so-sensoroni-start @@ -0,0 +1,12 @@ +#!/bin/bash + +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + + + +. /usr/sbin/so-common + +/usr/sbin/so-start sensoroni $1 diff --git a/salt/common/tools/sbin/so-sensoroni-stop b/salt/common/tools/sbin/so-sensoroni-stop new file mode 100755 index 000000000..5bec5b8d3 --- /dev/null +++ b/salt/common/tools/sbin/so-sensoroni-stop @@ -0,0 +1,12 @@ +#!/bin/bash + +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + + + +. /usr/sbin/so-common + +/usr/sbin/so-stop sensoroni $1 From 46aa7ebdf31d7cc02e4318e010901fe49d17628d Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 20 Jan 2023 06:48:33 -0500 Subject: [PATCH 12/19] correct find/exec syntax --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index d9c9bc2b2..416656bc2 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1943,7 +1943,7 @@ securityonion_repo() { # if the package is updated when the update_packages function is called logCmd "yum -v -y update centos-release" info "Backing up the .repo files that were added by the centos-release package." - logCmd "find /etc/yum.repos.d/ -type f -not -name 'securityonion*repo' -exec mv -bvf {} /root/oldrepos/ \;" + logCmd "find /etc/yum.repos.d/ -type f -not -name 'securityonion*repo' -exec mv -bvf {} /root/oldrepos/ \\;" logCmd "yum repolist all" fi } From ece63b72e2b957dae515c58e6185887e77542301 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 20 Jan 2023 07:38:58 -0500 Subject: [PATCH 13/19] Ensure so-verify output is logged --- setup/so-setup | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index fb19a9120..8eae60299 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -585,7 +585,7 @@ if ! [[ -f $install_opt_file ]]; then fi checkin_at_boot set_initial_firewall_access - ./so-verify $setup_type + logCmd ./so-verify $setup_type else touch /root/accept_changes mkdir -p /opt/so @@ -608,7 +608,7 @@ if ! [[ -f $install_opt_file ]]; then configure_minion "$minion_type" drop_install_options checkin_at_boot - ./so-verify $setup_type + logCmd ./so-verify $setup_type fi # Need to make sure the latest install is located on the web server of the manager to check the versions and donwload the code if required From 1e4f9c9f269c0023447fe9f7be1678c639b578cb Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 20 Jan 2023 11:01:02 -0500 Subject: [PATCH 14/19] use newer find syntax to allow the exec to work inside a quoted string --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 416656bc2..d804bef15 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1943,7 +1943,7 @@ securityonion_repo() { # if the package is updated when the update_packages function is called logCmd "yum -v -y update centos-release" info "Backing up the .repo files that were added by the centos-release package." - logCmd "find /etc/yum.repos.d/ -type f -not -name 'securityonion*repo' -exec mv -bvf {} /root/oldrepos/ \\;" + logCmd "find /etc/yum.repos.d/ -type f -not -name 'securityonion*repo' -exec mv -bvf {} /root/oldrepos/ +" logCmd "yum repolist all" fi } From c3384d838199e3758af07b4620538de7b6a51641 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 20 Jan 2023 11:23:13 -0500 Subject: [PATCH 15/19] further improvements --- setup/so-functions | 8 ++++++++ setup/so-setup | 6 ++---- setup/so-verify | 17 +---------------- 3 files changed, 11 insertions(+), 20 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index d804bef15..57f112982 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2380,3 +2380,11 @@ wait_for_file() { wait_for_salt_minion() { retry 60 5 "journalctl -u salt-minion.service | grep 'Minion is ready to receive requests'" >> "$setup_log" 2>&1 || exit 1 } + +verify_setup() { + if logCmd ./so-verify "$setup_type"; then + whiptail_setup_complete + else + whiptail_setup_failed + fi +} \ No newline at end of file diff --git a/setup/so-setup b/setup/so-setup index 8eae60299..2b0abedc8 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -585,7 +585,7 @@ if ! [[ -f $install_opt_file ]]; then fi checkin_at_boot set_initial_firewall_access - logCmd ./so-verify $setup_type + verify_setup else touch /root/accept_changes mkdir -p /opt/so @@ -608,10 +608,8 @@ if ! [[ -f $install_opt_file ]]; then configure_minion "$minion_type" drop_install_options checkin_at_boot - logCmd ./so-verify $setup_type + verify_setup fi # Need to make sure the latest install is located on the web server of the manager to check the versions and donwload the code if required - - fi diff --git a/setup/so-verify b/setup/so-verify index 234ef2dec..685e50d81 100755 --- a/setup/so-verify +++ b/setup/so-verify @@ -9,15 +9,10 @@ cd "$(dirname "$0")" || exit 255 source ../salt/common/tools/sbin/so-common source ./so-functions -source ./so-whiptail source ./so-variables setup_type=$1 -setup_in_progress() { - ps -ef | grep so-setup | grep -v grep &> /dev/null -} - using_iso() { if [ "$setup_type" == "iso" ]; then return 0 @@ -25,14 +20,6 @@ using_iso() { return 0 } -whipit() { - if [[ $exit_code -eq 0 ]]; then - whiptail_setup_complete - else - whiptail_setup_failed - fi -} - # Check entire setup log for errors or unexpected salt states log_has_errors() { # Ignore salt mast cached public key and minion failed to auth because this is a test @@ -44,7 +31,7 @@ log_has_errors() { # Ignore Failed: 0 since that is the salt state output, and we detect state failures # via Result: False already. - + grep -E "FAILED|Failed|failed|ERROR|Error|Result: False" "$setup_log" | \ grep -vE "The Salt Master has cached the public key for this node" | \ grep -vE "Minion failed to authenticate with the master" | \ @@ -105,8 +92,6 @@ main() { echo "Successfully completed setup!" fi - setup_in_progress && whipit $exit_code - exit $exit_code } From 56478da0b29b788c8a19bc4a1e56ddd14cf7abf3 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 20 Jan 2023 11:58:29 -0500 Subject: [PATCH 16/19] eliminate find/exec issue altogether to keep it simple --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 57f112982..03221d0e9 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1943,7 +1943,7 @@ securityonion_repo() { # if the package is updated when the update_packages function is called logCmd "yum -v -y update centos-release" info "Backing up the .repo files that were added by the centos-release package." - logCmd "find /etc/yum.repos.d/ -type f -not -name 'securityonion*repo' -exec mv -bvf {} /root/oldrepos/ +" + logCmd "mv -bvf /etc/yum.repos.d/CentOS* /root/oldrepos/" logCmd "yum repolist all" fi } From 95412140734c6254777fc81d137157b649947129 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 20 Jan 2023 12:26:52 -0500 Subject: [PATCH 17/19] logCmd with tee is eating the exit code --- setup/so-functions | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 03221d0e9..a046b49ca 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2382,7 +2382,11 @@ wait_for_salt_minion() { } verify_setup() { - if logCmd ./so-verify "$setup_type"; then + info "Verifying setup" + output=$(./so-verify "$setup_type" 2>&1) + result=$? + echo "$output" >> "$setup_log" + if [[ $result -eq 0 ]]; then whiptail_setup_complete else whiptail_setup_failed From 1ab8c712e492427ce121cd33ab58aabec5848551 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Fri, 20 Jan 2023 15:17:04 -0500 Subject: [PATCH 18/19] remove exit condition --- salt/playbook/files/automation_user_create.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/salt/playbook/files/automation_user_create.sh b/salt/playbook/files/automation_user_create.sh index bc827fda6..7279d13cc 100644 --- a/salt/playbook/files/automation_user_create.sh +++ b/salt/playbook/files/automation_user_create.sh @@ -2,8 +2,6 @@ # {%- set admin_pass = salt['pillar.get']('secrets:playbook_admin', None) -%} # {%- set automation_pass = salt['pillar.get']('secrets:playbook_automation', None) %} -set -e - local_salt_dir=/opt/so/saltstack/local try_count=6 From d3cb57bba2dd2745c737f3de4a6458531c3fc5cf Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Mon, 23 Jan 2023 08:16:28 -0500 Subject: [PATCH 19/19] Rerun the playbook state --- salt/common/tools/sbin/so-playbook-reset | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-playbook-reset b/salt/common/tools/sbin/so-playbook-reset index 9907e1807..d58078997 100755 --- a/salt/common/tools/sbin/so-playbook-reset +++ b/salt/common/tools/sbin/so-playbook-reset @@ -9,7 +9,11 @@ . /usr/sbin/so-common -salt-call state.apply playbook.db_init,playbook,playbook.automation_user_create +salt-call state.apply playbook.db_init,playbook + +/usr/sbin/so-soctopus-restart + +salt-call state.apply playbook,playbook.automation_user_create /usr/sbin/so-soctopus-restart