From 268e07e2a24d4848fb2555f5b6517b0fa7e81f9e Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 24 Jan 2022 15:49:55 -0500 Subject: [PATCH 1/6] remove jinja from soup scripts --- salt/common/init.sls | 8 +++++- salt/common/soup_scripts.sls | 13 +++++++++ salt/common/tools/sbin/so-common | 4 --- salt/common/tools/sbin/soup | 46 ++++++-------------------------- 4 files changed, 28 insertions(+), 43 deletions(-) create mode 100644 salt/common/soup_scripts.sls diff --git a/salt/common/init.sls b/salt/common/init.sls index e511308a7..d1acca878 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -4,8 +4,9 @@ {% set role = grains.id.split('_') | last %} {% from 'elasticsearch/auth.map.jinja' import ELASTICAUTH with context %} -{% if grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-managersearch', 'so-import'] %} include: + - common.soup_scripts +{% if grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-managersearch', 'so-import'] %} - manager.elasticsearch # needed for elastic_curl_config state {% endif %} @@ -214,6 +215,11 @@ utilsyncscripts: ELASTICCURL: 'curl' - context: ELASTICCURL: {{ ELASTICAUTH.elasticcurl }} + - exclude_pat: + - so-common + - so-firewall + - so-image-common + - soup {% if role in ['eval', 'standalone', 'sensor', 'heavynode'] %} # Add sensor cleanup diff --git a/salt/common/soup_scripts.sls b/salt/common/soup_scripts.sls new file mode 100644 index 000000000..ed3b8696c --- /dev/null +++ b/salt/common/soup_scripts.sls @@ -0,0 +1,13 @@ +# Sync some Utilities +soup_scripts: + file.recurse: + - name: /usr/sbin + - user: root + - group: root + - file_mode: 755 + - source: salt://common/tools/sbin + - include_pat: + - so-common + - so-firewall + - so-image-common + - soup diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index d50ec0672..64dac566d 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -517,8 +517,6 @@ valid_int() { [[ $num =~ ^[0-9]*$ ]] && [[ $num -ge $min ]] && [[ $num -le $max ]] && return 0 || return 1 } -# {% raw %} - valid_proxy() { local proxy=$1 local url_prefixes=( 'http://' 'https://' ) @@ -561,8 +559,6 @@ valid_string() { echo "$str" | grep -qP '^\S+$' && [[ ${#str} -ge $min_length ]] && [[ ${#str} -le $max_length ]] && return 0 || return 1 } -# {% endraw %} - valid_username() { local user=$1 diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 9b305fc76..6afff4a81 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -195,8 +195,6 @@ check_airgap() { fi } -# {% raw %} - check_local_mods() { local salt_local=/opt/so/saltstack/local @@ -224,8 +222,6 @@ check_local_mods() { fi } -# {% endraw %} - check_pillar_items() { local pillar_output=$(salt-call pillar.items --out=json) @@ -663,9 +659,6 @@ up_to_2.3.90() { fi done - # There was a bug in 2.3.0 so-firewall addhostgroup that was resolved in 2.3.1 - commit 32294eb2ed30ac74b15bb4bfab687084a928daf2 - echo "Verify so-firewall is up to date" - verify_latest_so-firewall_script # Create Endgame Hostgroup echo "Adding endgame hostgroup with so-firewall" if so-firewall addhostgroup endgame 2>&1 | grep -q 'Already exists'; then @@ -889,47 +882,24 @@ update_repo() { } verify_latest_update_script() { - #we need to render soup and so-common first since they contain jinja - salt-call slsutil.renderer $UPDATE_DIR/salt/common/tools/sbin/soup default_renderer='jinja' --local --out=newline_values_only --out-indent=-4 --out-file=/tmp/soup - sed -i -e '$a\' /tmp/soup - salt-call slsutil.renderer $UPDATE_DIR/salt/common/tools/sbin/so-common default_renderer='jinja' --local --out=newline_values_only --out-indent=-4 --out-file=/tmp/so-common - sed -i -e '$a\' /tmp/so-common # Check to see if the update scripts match. If not run the new one. CURRENTSOUP=$(md5sum /usr/sbin/soup | awk '{print $1}') - GITSOUP=$(md5sum /tmp/soup | awk '{print $1}') + GITSOUP=$(md5sum $UPDATE_DIR/salt/common/tools/sbin/soup | awk '{print $1}') CURRENTCMN=$(md5sum /usr/sbin/so-common | awk '{print $1}') - GITCMN=$(md5sum /tmp/so-common | awk '{print $1}') + GITCMN=$(md5sum $UPDATE_DIR/salt/common/tools/sbin/so-common | awk '{print $1}') CURRENTIMGCMN=$(md5sum /usr/sbin/so-image-common | awk '{print $1}') GITIMGCMN=$(md5sum $UPDATE_DIR/salt/common/tools/sbin/so-image-common | awk '{print $1}') - - if [[ "$CURRENTSOUP" == "$GITSOUP" && "$CURRENTCMN" == "$GITCMN" && "$CURRENTIMGCMN" == "$GITIMGCMN" ]]; then - echo "This version of the soup script is up to date. Proceeding." - rm -f /tmp/soup /tmp/so-common - else - echo "You are not running the latest soup version. Updating soup and its components. Might take multiple runs to complete" - cp $UPDATE_DIR/salt/common/tools/sbin/soup $DEFAULT_SALT_DIR/salt/common/tools/sbin/ - cp $UPDATE_DIR/salt/common/tools/sbin/so-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/ - cp $UPDATE_DIR/salt/common/tools/sbin/so-image-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/ - salt-call state.apply -l info common queue=True - echo "" - echo "soup has been updated. Please run soup again." - exit 0 - fi -} - -verify_latest_so-firewall_script() { - # Check to see if the so-firewall script matches. If not run the new one. CURRENTSOFIREWALL=$(md5sum /usr/sbin/so-firewall | awk '{print $1}') GITSOFIREWALL=$(md5sum $UPDATE_DIR/salt/common/tools/sbin/so-firewall | awk '{print $1}') - if [[ "$CURRENTSOFIREWALL" == "$GITSOFIREWALL" ]]; then - echo "This version of the so-firewall script is up to date. Proceeding." + if [[ "$CURRENTSOUP" == "$GITSOUP" && "$CURRENTCMN" == "$GITCMN" && "$CURRENTIMGCMN" == "$GITIMGCMN" && "$CURRENTSOFIREWALL" == "$GITSOFIREWALL" ]]; then + echo "This version of the soup script is up to date. Proceeding." else - echo "You are not running the latest version of so-firewall. Updating so-firewall." - cp $UPDATE_DIR/salt/common/tools/sbin/so-firewall $DEFAULT_SALT_DIR/salt/common/tools/sbin/ - cp $UPDATE_DIR/salt/common/tools/sbin/so-firewall /usr/sbin/ + echo "You are not running the latest soup version. Updating soup and its components. Might take multiple runs to complete" + salt-call state.apply common.soup_scripts queue=True -linfo --file-root=$UPDATE_DIR/salt echo "" - echo "so-firewall has been updated." + echo "soup has been updated. Please run soup again." + exit 0 fi } From e3f1b456e6722b2146add8bf695f5c4fcadc6f44 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 24 Jan 2022 16:09:15 -0500 Subject: [PATCH 2/6] add raw end raw back --- salt/common/tools/sbin/so-common | 4 ++++ salt/common/tools/sbin/soup | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 64dac566d..d50ec0672 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -517,6 +517,8 @@ valid_int() { [[ $num =~ ^[0-9]*$ ]] && [[ $num -ge $min ]] && [[ $num -le $max ]] && return 0 || return 1 } +# {% raw %} + valid_proxy() { local proxy=$1 local url_prefixes=( 'http://' 'https://' ) @@ -559,6 +561,8 @@ valid_string() { echo "$str" | grep -qP '^\S+$' && [[ ${#str} -ge $min_length ]] && [[ ${#str} -le $max_length ]] && return 0 || return 1 } +# {% endraw %} + valid_username() { local user=$1 diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 6afff4a81..9e9535322 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -195,6 +195,8 @@ check_airgap() { fi } +# {% raw %} + check_local_mods() { local salt_local=/opt/so/saltstack/local @@ -222,6 +224,8 @@ check_local_mods() { fi } +# {% endraw %} + check_pillar_items() { local pillar_output=$(salt-call pillar.items --out=json) From d083338350ecb4797eb42da7362a53611c9baac1 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 24 Jan 2022 16:46:29 -0500 Subject: [PATCH 3/6] adding --local --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 9e9535322..d85d76d8c 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -900,7 +900,7 @@ verify_latest_update_script() { echo "This version of the soup script is up to date. Proceeding." else echo "You are not running the latest soup version. Updating soup and its components. Might take multiple runs to complete" - salt-call state.apply common.soup_scripts queue=True -linfo --file-root=$UPDATE_DIR/salt + salt-call state.apply common.soup_scripts queue=True -linfo --file-root=$UPDATE_DIR/salt --local echo "" echo "soup has been updated. Please run soup again." exit 0 From 82e2b2b611667261c7196d5420d794a926875330 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 24 Jan 2022 17:03:25 -0500 Subject: [PATCH 4/6] dont escape raw and endraw --- salt/common/tools/sbin/soup | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index d85d76d8c..8b90e8a5a 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -683,8 +683,8 @@ up_to_2.3.90() { if [[ -f "/opt/so/saltstack/local/salt/elasticsearch/files/ingest/common" ]]; then mv -v /opt/so/saltstack/local/salt/elasticsearch/files/ingest/common /opt/so/saltstack/local/salt/elasticsearch/files/ingest-dynamic/common # since json file, we need to wrap with raw - sed -i '1s/^/{{'{% raw %}'}}\n/' /opt/so/saltstack/local/salt/elasticsearch/files/ingest-dynamic/common - sed -i -e '$a{{'{% endraw %}'}}\n' /opt/so/saltstack/local/salt/elasticsearch/files/ingest-dynamic/common + sed -i '1s/^/{% raw %}\n/' /opt/so/saltstack/local/salt/elasticsearch/files/ingest-dynamic/common + sed -i -e '$a{% endraw %}\n' /opt/so/saltstack/local/salt/elasticsearch/files/ingest-dynamic/common fi # Generate FleetDM Service Account creds if they do not exist From d6fc436d4991dfced96a8c316db1143a38f8c2e8 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 24 Jan 2022 19:30:34 -0500 Subject: [PATCH 5/6] copy files to default salt base --- salt/common/tools/sbin/soup | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 8b90e8a5a..b3f9cd1ca 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -900,6 +900,10 @@ verify_latest_update_script() { echo "This version of the soup script is up to date. Proceeding." else echo "You are not running the latest soup version. Updating soup and its components. Might take multiple runs to complete" + cp $UPDATE_DIR/salt/common/tools/sbin/soup $DEFAULT_SALT_DIR/salt/common/tools/sbin/ + cp $UPDATE_DIR/salt/common/tools/sbin/so-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/ + cp $UPDATE_DIR/salt/common/tools/sbin/so-image-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/ + cp $UPDATE_DIR/salt/common/tools/sbin/so-firewall $DEFAULT_SALT_DIR/salt/common/tools/sbin/ salt-call state.apply common.soup_scripts queue=True -linfo --file-root=$UPDATE_DIR/salt --local echo "" echo "soup has been updated. Please run soup again." From edd8709cdd63cec59360942c15fcfec0958f19f6 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 24 Jan 2022 19:42:56 -0500 Subject: [PATCH 6/6] remove export LC_CTYPE="en_US.UTF-8" from soup --- salt/common/tools/sbin/soup | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index b3f9cd1ca..dd4538309 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -16,7 +16,6 @@ # along with this program. If not, see . . /usr/sbin/so-common -export LC_CTYPE="en_US.UTF-8" UPDATE_DIR=/tmp/sogh/securityonion DEFAULT_SALT_DIR=/opt/so/saltstack/default