From cd6e387bcbc3a21bb8bd2121221496345ec9ec6e Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Thu, 25 Jan 2024 16:15:53 -0500 Subject: [PATCH 01/25] remove --local from soup common.soup_scripts update. Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/manager/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 0fc4f75d8..e66d15a56 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -776,7 +776,7 @@ verify_latest_update_script() { cp $UPDATE_DIR/salt/common/tools/sbin/so-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/ cp $UPDATE_DIR/salt/common/tools/sbin/so-image-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/ cp $UPDATE_DIR/salt/manager/tools/sbin/so-firewall $DEFAULT_SALT_DIR/salt/common/tools/sbin/ - salt-call state.apply common.soup_scripts queue=True -linfo --file-root=$UPDATE_DIR/salt --local + salt-call state.apply common.soup_scripts queue=True -linfo echo "" echo "The soup script has been modified. Please run soup again to continue the upgrade." exit 0 From 2e026b637d046bc8444bb5e4db9843f87709ec00 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Fri, 26 Jan 2024 11:36:33 -0500 Subject: [PATCH 02/25] Update soup to retry modified salt command on failure to update soup scripts. Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/manager/tools/sbin/soup | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index e66d15a56..948fc10b1 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -372,6 +372,17 @@ enable_highstate() { echo "" } +get_soup_script_hashes() { + CURRENTSOUP=$(md5sum /usr/sbin/soup | awk '{print $1}') + GITSOUP=$(md5sum $UPDATE_DIR/salt/manager/tools/sbin/soup | awk '{print $1}') + CURRENTCMN=$(md5sum /usr/sbin/so-common | awk '{print $1}') + GITCMN=$(md5sum $UPDATE_DIR/salt/common/tools/sbin/so-common | awk '{print $1}') + CURRENTIMGCMN=$(md5sum /usr/sbin/so-image-common | awk '{print $1}') + GITIMGCMN=$(md5sum $UPDATE_DIR/salt/common/tools/sbin/so-image-common | awk '{print $1}') + CURRENTSOFIREWALL=$(md5sum /usr/sbin/so-firewall | awk '{print $1}') + GITSOFIREWALL=$(md5sum $UPDATE_DIR/salt/manager/tools/sbin/so-firewall | awk '{print $1}') +} + highstate() { # Run a highstate. salt-call state.highstate -l info queue=True @@ -758,16 +769,7 @@ upgrade_salt() { } verify_latest_update_script() { - # Check to see if the update scripts match. If not run the new one. - CURRENTSOUP=$(md5sum /usr/sbin/soup | awk '{print $1}') - GITSOUP=$(md5sum $UPDATE_DIR/salt/manager/tools/sbin/soup | awk '{print $1}') - CURRENTCMN=$(md5sum /usr/sbin/so-common | awk '{print $1}') - GITCMN=$(md5sum $UPDATE_DIR/salt/common/tools/sbin/so-common | awk '{print $1}') - CURRENTIMGCMN=$(md5sum /usr/sbin/so-image-common | awk '{print $1}') - GITIMGCMN=$(md5sum $UPDATE_DIR/salt/common/tools/sbin/so-image-common | awk '{print $1}') - CURRENTSOFIREWALL=$(md5sum /usr/sbin/so-firewall | awk '{print $1}') - GITSOFIREWALL=$(md5sum $UPDATE_DIR/salt/manager/tools/sbin/so-firewall | awk '{print $1}') - + get_soup_script_hashes if [[ "$CURRENTSOUP" == "$GITSOUP" && "$CURRENTCMN" == "$GITCMN" && "$CURRENTIMGCMN" == "$GITIMGCMN" && "$CURRENTSOFIREWALL" == "$GITSOFIREWALL" ]]; then echo "This version of the soup script is up to date. Proceeding." else @@ -776,7 +778,13 @@ verify_latest_update_script() { cp $UPDATE_DIR/salt/common/tools/sbin/so-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/ cp $UPDATE_DIR/salt/common/tools/sbin/so-image-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/ cp $UPDATE_DIR/salt/manager/tools/sbin/so-firewall $DEFAULT_SALT_DIR/salt/common/tools/sbin/ - salt-call state.apply common.soup_scripts queue=True -linfo + salt-call state.apply common.soup_scripts queue=True -linfo --file-root=$UPDATE_DIR/salt --local + # Verify that soup scripts updated as expected + get_soup_script_hashes + if [[ "$CURRENTSOUP" == "$GITSOUP" && "$CURRENTCMN" == "$GITCMN" && "$CURRENTIMGCMN" == "$GITIMGCMN" && "$CURRENTSOFIREWALL" == "$GITSOFIREWALL" ]]; then + echo "There was a problem updating soup scripts.. Trying to rerun script update" + salt-call state.apply common.soup_scripts queue=True -linfo + else echo "" echo "The soup script has been modified. Please run soup again to continue the upgrade." exit 0 From 91c7b8144d28e5ffb01b00c017bcbd429afd50f2 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Fri, 26 Jan 2024 15:43:42 -0500 Subject: [PATCH 03/25] soup logic Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/manager/tools/sbin/soup | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 948fc10b1..2e9c423ff 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -782,9 +782,11 @@ verify_latest_update_script() { # Verify that soup scripts updated as expected get_soup_script_hashes if [[ "$CURRENTSOUP" == "$GITSOUP" && "$CURRENTCMN" == "$GITCMN" && "$CURRENTIMGCMN" == "$GITIMGCMN" && "$CURRENTSOFIREWALL" == "$GITSOFIREWALL" ]]; then + echo "Succesfully updated soup scripts." + else echo "There was a problem updating soup scripts.. Trying to rerun script update" salt-call state.apply common.soup_scripts queue=True -linfo - else + fi echo "" echo "The soup script has been modified. Please run soup again to continue the upgrade." exit 0 From c4301d7cc1deeb426604b248de24096243c4a248 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Fri, 26 Jan 2024 15:51:06 -0500 Subject: [PATCH 04/25] Soup script update locations Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/manager/tools/sbin/soup | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 2e9c423ff..86cb709dd 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -774,10 +774,10 @@ verify_latest_update_script() { echo "This version of the soup script is up to date. Proceeding." else echo "You are not running the latest soup version. Updating soup and its components. This might take multiple runs to complete." - cp $UPDATE_DIR/salt/manager/tools/sbin/soup $DEFAULT_SALT_DIR/salt/common/tools/sbin/ + cp $UPDATE_DIR/salt/manager/tools/sbin/soup $DEFAULT_SALT_DIR/salt/manager/tools/sbin/ cp $UPDATE_DIR/salt/common/tools/sbin/so-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/ cp $UPDATE_DIR/salt/common/tools/sbin/so-image-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/ - cp $UPDATE_DIR/salt/manager/tools/sbin/so-firewall $DEFAULT_SALT_DIR/salt/common/tools/sbin/ + cp $UPDATE_DIR/salt/manager/tools/sbin/so-firewall $DEFAULT_SALT_DIR/salt/manager/tools/sbin/ salt-call state.apply common.soup_scripts queue=True -linfo --file-root=$UPDATE_DIR/salt --local # Verify that soup scripts updated as expected get_soup_script_hashes @@ -791,8 +791,8 @@ verify_latest_update_script() { echo "The soup script has been modified. Please run soup again to continue the upgrade." exit 0 fi -} +} # Keeping this block in case we need to do a hotfix that requires salt update apply_hotfix() { if [[ "$INSTALLEDVERSION" == "2.4.20" ]] ; then From 7c08b348aafb9ee9a842e67356e4b09096217fbb Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 29 Jan 2024 10:16:34 -0500 Subject: [PATCH 05/25] Add comment for soup update w/ STIGs enabled Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/manager/tools/sbin/soup | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 86cb709dd..a250116d1 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -784,6 +784,8 @@ verify_latest_update_script() { if [[ "$CURRENTSOUP" == "$GITSOUP" && "$CURRENTCMN" == "$GITCMN" && "$CURRENTIMGCMN" == "$GITIMGCMN" && "$CURRENTSOFIREWALL" == "$GITSOFIREWALL" ]]; then echo "Succesfully updated soup scripts." else + # When STIGs are enabled soup scripts will fail to update using --file-root --local. + # After checking that the expected hashes are not present, retry updating soup scripts using salt master. echo "There was a problem updating soup scripts.. Trying to rerun script update" salt-call state.apply common.soup_scripts queue=True -linfo fi From b5ffa186fb35f104f9e7d6bca954b5a9af3df442 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Tue, 30 Jan 2024 15:54:23 -0500 Subject: [PATCH 06/25] Remove remediate from initial oscap scan Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/stig/enabled.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/stig/enabled.sls b/salt/stig/enabled.sls index 5c4b6851b..5f6c72c92 100644 --- a/salt/stig/enabled.sls +++ b/salt/stig/enabled.sls @@ -50,7 +50,7 @@ update_stig_profile: run_initial_scan: module.run: - name: openscap.xccdf - - params: 'eval --remediate --profile {{ OSCAP_PROFILE_NAME }} --results {{ OSCAP_OUTPUT_DIR }}/pre-oscap-results.xml --report {{ OSCAP_OUTPUT_DIR }}/pre-oscap-report.html {{ OSCAP_PROFILE_LOCATION }}' + - params: 'eval --profile {{ OSCAP_PROFILE_NAME }} --results {{ OSCAP_OUTPUT_DIR }}/pre-oscap-results.xml --report {{ OSCAP_OUTPUT_DIR }}/pre-oscap-report.html {{ OSCAP_PROFILE_LOCATION }}' {% endif %} run_remediate: From cd4bd6460aa117ebc798c6f74f5ddfa32b6fc65f Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 31 Jan 2024 20:16:18 +0000 Subject: [PATCH 07/25] Custom pipelines --- salt/elasticsearch/defaults.yaml | 81 ++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index ce1bfb08d..541f3fae7 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -55,6 +55,87 @@ elasticsearch: key: /usr/share/elasticsearch/config/elasticsearch.key verification_mode: none enabled: false + pipelines: + custom01: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom01 + - pipeline: + name: common + custom02: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom02 + - pipeline: + name: common + custom03: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom03 + - pipeline: + name: common + custom04: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom04 + - pipeline: + name: common + custom05: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom05 + - pipeline: + name: common + custom06: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom06 + - pipeline: + name: common + custom07: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom07 + - pipeline: + name: common + custom08: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom08 + - pipeline: + name: common + custom09: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom09 + - pipeline: + name: common + custom10: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom10 + - pipeline: + name: common index_settings: global_overrides: index_template: From bc75be940243793ee0b33c031efbcc00c2c988e3 Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 31 Jan 2024 20:16:48 +0000 Subject: [PATCH 08/25] Custom pipelines in UI --- salt/elasticsearch/config.sls | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/salt/elasticsearch/config.sls b/salt/elasticsearch/config.sls index 23e11a710..27a8a0fd6 100644 --- a/salt/elasticsearch/config.sls +++ b/salt/elasticsearch/config.sls @@ -118,6 +118,19 @@ esingestconf: - user: 930 - group: 939 +# Auto-generate Elasticsearch ingest node pipelines from pillar +{% for pipeline, config in ELASTICSEARCHMERGED.pipelines.items() %} +es_ingest_conf_{{pipeline}}: + file.managed: + - name: /opt/so/conf/elasticsearch/ingest/{{ pipeline }} + - source: salt://elasticsearch/base-template.json.jinja + - defaults: + TEMPLATE_CONFIG: {{ config }} + - template: jinja + - onchanges_in: + - file: so-pipelines-reload +{% endfor %} + eslog4jfile: file.managed: - name: /opt/so/conf/elasticsearch/log4j2.properties From 1853dc398bc377dc367cd0aaeeaefece7acec3c5 Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 31 Jan 2024 20:17:33 +0000 Subject: [PATCH 09/25] Custom pipeline configuration --- salt/logstash/defaults.yaml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/salt/logstash/defaults.yaml b/salt/logstash/defaults.yaml index e4c18cc64..2cafce6fd 100644 --- a/salt/logstash/defaults.yaml +++ b/salt/logstash/defaults.yaml @@ -42,6 +42,24 @@ logstash: custom2: [] custom3: [] custom4: [] + pipeline_config: + custom01: |- + filter { + if [event][module] =~ "zeek" { + mutate { + add_tag => ["network_stuff"] + } + } + } + custom02: PLACEHOLDER + custom03: PLACEHOLDER + custom04: PLACEHOLDER + custom05: PLACEHOLDER + custom06: PLACEHOLDER + custom07: PLACEHOLDER + custom08: PLACEHOLDER + custom09: PLACEHOLDER + custom10: PLACEHOLDER settings: lsheap: 500m config: From 4672a5b8ebe83e4c5abac79abf5e87ab3c1f3d39 Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 31 Jan 2024 20:18:17 +0000 Subject: [PATCH 10/25] Custom pipeline configuration in UI --- salt/logstash/config.sls | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/salt/logstash/config.sls b/salt/logstash/config.sls index 6ff33ff1a..8a59c83b7 100644 --- a/salt/logstash/config.sls +++ b/salt/logstash/config.sls @@ -63,6 +63,20 @@ lspipelinedir: - user: 931 - group: 939 +# Auto-generate Logstash pipeline config +{% for pipeline, config in LOGSTASH_MERGED.pipeline_config.items() %} +{% for assigned_pipeline in ASSIGNED_PIPELINES %} +{% set custom_pipeline = 'custom/' + pipeline + '.conf' %} +{% if custom_pipeline in LOGSTASH_MERGED.defined_pipelines[assigned_pipeline] %} +ls_custom_pipeline_conf_{{assigned_pipeline}}_{{pipeline}}: + file.managed: + - name: /opt/so/conf/logstash/pipelines/{{assigned_pipeline}}/{{ pipeline }}.conf + - contents: LOGSTASH_MERGED.pipeline_config.{{pipeline}} +{% endif %} +{% endfor %} +{% endfor %} + + {% for assigned_pipeline in ASSIGNED_PIPELINES %} {% for CONFIGFILE in LOGSTASH_MERGED.defined_pipelines[assigned_pipeline] %} ls_pipeline_{{assigned_pipeline}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }}: From 341ff5b56436289ca9614c08dea10134d222f239 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 31 Jan 2024 16:18:51 -0500 Subject: [PATCH 11/25] Update so-functions --- setup/so-functions | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index d19f27620..46bb0b4b2 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1933,7 +1933,11 @@ saltify() { logCmd "dnf -y install salt-$SALTVERSION salt-master-$SALTVERSION salt-minion-$SALTVERSION" else # We just need the minion - logCmd "dnf -y install salt-$SALTVERSION salt-minion-$SALTVERSION" + if [[ $is_airgap ]]; then + logCmd "dnf -y install salt salt-minion" + else + logCmd "dnf -y install salt-$SALTVERSION salt-minion-$SALTVERSION" + fi fi fi From bc502cc065c934d1afad9c08c4001a5bcb9dd732 Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 31 Jan 2024 21:46:33 +0000 Subject: [PATCH 12/25] Custom Elasticserach pipeline annotations --- salt/elasticsearch/soc_elasticsearch.yaml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index c54e07660..0f410e716 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -45,6 +45,28 @@ elasticsearch: description: Max number of boolean clauses per query. global: True helpLink: elasticsearch.html + pipelines: + custom01: &pipelines + description: + description: Description of the ingest node pipeline + global: True + advanced: True + helpLink: elasticsearch.html + processors: + description: Processors for the ingest node pipeline + global: True + advanced: True + multiline: True + helpLink: elasticsearch.html + custom02: *pipelines + custom03: *pipelines + custom04: *pipelines + custom05: *pipelines + custom06: *pipelines + custom07: *pipelines + custom08: *pipelines + custom09: *pipelines + custom10: *pipelines index_settings: global_overrides: index_template: From 136097f9816b30b85804a2b43ddf208205c28581 Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 31 Jan 2024 21:47:09 +0000 Subject: [PATCH 13/25] Custom Logstash pipeline annotations --- salt/logstash/soc_logstash.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/salt/logstash/soc_logstash.yaml b/salt/logstash/soc_logstash.yaml index bcb99bad5..dc52d63a8 100644 --- a/salt/logstash/soc_logstash.yaml +++ b/salt/logstash/soc_logstash.yaml @@ -31,6 +31,22 @@ logstash: custom2: *defined_pipelines custom3: *defined_pipelines custom4: *defined_pipelines + pipeline_config: + custom01: &pipeline_config + description: Pipeline configuration for Logstash + advanced: True + multiline: True + forcedType: string + helpLink: logstash.html + custom02: *pipeline_config + custom03: *pipeline_config + custom04: *pipeline_config + custom05: *pipeline_config + custom06: *pipeline_config + custom07: *pipeline_config + custom08: *pipeline_config + custom09: *pipeline_config + custom10: *pipeline_config settings: lsheap: description: Heap size to use for logstash From 182667bafb3769b636f715385d24805dda40c2bb Mon Sep 17 00:00:00 2001 From: Wes Date: Thu, 1 Feb 2024 13:59:23 +0000 Subject: [PATCH 14/25] Change numbers for Elasticsearch --- salt/elasticsearch/defaults.yaml | 40 +++++++++++------------ salt/elasticsearch/soc_elasticsearch.yaml | 20 ++++++------ 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 541f3fae7..03cd6d519 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -56,84 +56,84 @@ elasticsearch: verification_mode: none enabled: false pipelines: - custom01: + custom001: description: Custom Pipeline processors: - set: field: tags - value: custom01 + value: custom001 - pipeline: name: common - custom02: + custom002: description: Custom Pipeline processors: - set: field: tags - value: custom02 + value: custom002 - pipeline: name: common - custom03: + custom003: description: Custom Pipeline processors: - set: field: tags - value: custom03 + value: custom003 - pipeline: name: common - custom04: + custom004: description: Custom Pipeline processors: - set: field: tags - value: custom04 + value: custom004 - pipeline: name: common - custom05: + custom005: description: Custom Pipeline processors: - set: field: tags - value: custom05 + value: custom005 - pipeline: name: common - custom06: + custom006: description: Custom Pipeline processors: - set: field: tags - value: custom06 + value: custom006 - pipeline: name: common - custom07: + custom007: description: Custom Pipeline processors: - set: field: tags - value: custom07 + value: custom007 - pipeline: name: common - custom08: + custom008: description: Custom Pipeline processors: - set: field: tags - value: custom08 + value: custom008 - pipeline: name: common - custom09: + custom009: description: Custom Pipeline processors: - set: field: tags - value: custom09 + value: custom009 - pipeline: name: common - custom10: + custom010: description: Custom Pipeline processors: - set: field: tags - value: custom10 + value: custom010 - pipeline: name: common index_settings: diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index 0f410e716..9a64190b3 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -46,7 +46,7 @@ elasticsearch: global: True helpLink: elasticsearch.html pipelines: - custom01: &pipelines + custom001: &pipelines description: description: Description of the ingest node pipeline global: True @@ -58,15 +58,15 @@ elasticsearch: advanced: True multiline: True helpLink: elasticsearch.html - custom02: *pipelines - custom03: *pipelines - custom04: *pipelines - custom05: *pipelines - custom06: *pipelines - custom07: *pipelines - custom08: *pipelines - custom09: *pipelines - custom10: *pipelines + custom002: *pipelines + custom003: *pipelines + custom004: *pipelines + custom005: *pipelines + custom006: *pipelines + custom007: *pipelines + custom008: *pipelines + custom009: *pipelines + custom010: *pipelines index_settings: global_overrides: index_template: From 1818e134cafe3c29224d66cd623f839d2065e8cd Mon Sep 17 00:00:00 2001 From: Wes Date: Thu, 1 Feb 2024 14:01:55 +0000 Subject: [PATCH 15/25] Change numbers for Logstash --- salt/logstash/defaults.yaml | 20 ++++++++++---------- salt/logstash/soc_logstash.yaml | 20 ++++++++++---------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/salt/logstash/defaults.yaml b/salt/logstash/defaults.yaml index 2cafce6fd..348acb622 100644 --- a/salt/logstash/defaults.yaml +++ b/salt/logstash/defaults.yaml @@ -43,7 +43,7 @@ logstash: custom3: [] custom4: [] pipeline_config: - custom01: |- + custom001: |- filter { if [event][module] =~ "zeek" { mutate { @@ -51,15 +51,15 @@ logstash: } } } - custom02: PLACEHOLDER - custom03: PLACEHOLDER - custom04: PLACEHOLDER - custom05: PLACEHOLDER - custom06: PLACEHOLDER - custom07: PLACEHOLDER - custom08: PLACEHOLDER - custom09: PLACEHOLDER - custom10: PLACEHOLDER + custom002: PLACEHOLDER + custom003: PLACEHOLDER + custom004: PLACEHOLDER + custom005: PLACEHOLDER + custom006: PLACEHOLDER + custom007: PLACEHOLDER + custom008: PLACEHOLDER + custom009: PLACEHOLDER + custom010: PLACEHOLDER settings: lsheap: 500m config: diff --git a/salt/logstash/soc_logstash.yaml b/salt/logstash/soc_logstash.yaml index dc52d63a8..3172ff7c5 100644 --- a/salt/logstash/soc_logstash.yaml +++ b/salt/logstash/soc_logstash.yaml @@ -32,21 +32,21 @@ logstash: custom3: *defined_pipelines custom4: *defined_pipelines pipeline_config: - custom01: &pipeline_config + custom001: &pipeline_config description: Pipeline configuration for Logstash advanced: True multiline: True forcedType: string helpLink: logstash.html - custom02: *pipeline_config - custom03: *pipeline_config - custom04: *pipeline_config - custom05: *pipeline_config - custom06: *pipeline_config - custom07: *pipeline_config - custom08: *pipeline_config - custom09: *pipeline_config - custom10: *pipeline_config + custom002: *pipeline_config + custom003: *pipeline_config + custom004: *pipeline_config + custom005: *pipeline_config + custom006: *pipeline_config + custom007: *pipeline_config + custom008: *pipeline_config + custom009: *pipeline_config + custom010: *pipeline_config settings: lsheap: description: Heap size to use for logstash From 0d5db58c86f39521ec954d186db6050b7d41fbad Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 1 Feb 2024 10:32:41 -0500 Subject: [PATCH 16/25] upgrade salt3006.6 --- salt/salt/master.defaults.yaml | 2 +- salt/salt/minion.defaults.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/salt/master.defaults.yaml b/salt/salt/master.defaults.yaml index 1b4d2e63a..19677f70b 100644 --- a/salt/salt/master.defaults.yaml +++ b/salt/salt/master.defaults.yaml @@ -1,4 +1,4 @@ # version cannot be used elsewhere in this pillar as soup is grepping for it to determine if Salt needs to be patched salt: master: - version: 3006.5 + version: 3006.6 diff --git a/salt/salt/minion.defaults.yaml b/salt/salt/minion.defaults.yaml index c15929951..2e4ebc93e 100644 --- a/salt/salt/minion.defaults.yaml +++ b/salt/salt/minion.defaults.yaml @@ -1,6 +1,6 @@ # version cannot be used elsewhere in this pillar as soup is grepping for it to determine if Salt needs to be patched salt: minion: - version: 3006.5 + version: 3006.6 check_threshold: 3600 # in seconds, threshold used for so-salt-minion-check. any value less than 600 seconds may cause a lot of salt-minion restarts since the job to touch the file occurs every 5-8 minutes by default service_start_delay: 30 # in seconds. From 2643ae08a75c097abc1de60397f33b72a66655f0 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 5 Feb 2024 17:54:30 -0500 Subject: [PATCH 17/25] add append to list --- salt/manager/tools/sbin/so-yaml.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/salt/manager/tools/sbin/so-yaml.py b/salt/manager/tools/sbin/so-yaml.py index 874fc9e0f..572585df9 100755 --- a/salt/manager/tools/sbin/so-yaml.py +++ b/salt/manager/tools/sbin/so-yaml.py @@ -16,12 +16,14 @@ lockFile = "/tmp/so-yaml.lock" def showUsage(args): print('Usage: {} [ARGS...]'.format(sys.argv[0])) print(' General commands:') + print(' append - Append a list item to a yaml key, if it exists and is a list. Requires KEY and LISTITEM args.') print(' remove - Removes a yaml key, if it exists. Requires KEY arg.') print(' help - Prints this usage information.') print('') print(' Where:') print(' YAML_FILE - Path to the file that will be modified. Ex: /opt/so/conf/service/conf.yaml') print(' KEY - YAML key, does not support \' or " characters at this time. Ex: level1.level2') + print(' LISTITEM - Item to add to the list.') sys.exit(1) @@ -35,6 +37,32 @@ def writeYaml(filename, content): file = open(filename, "w") return yaml.dump(content, file) +def appendItem(content, key, listItem): + pieces = key.split(".", 1) + if len(pieces) > 1: + appendItem(content[pieces[0]], pieces[1], listItem) + else: + try: + content[key].append(listItem) + except TypeError: + print("The contents key provided is likely not a list. No action was taken on the file.") + return 1 + +def append(args): + if len(args) != 3: + print('Missing filename, key arg, or list item to append', file=sys.stderr) + showUsage(None) + return + + filename = args[0] + key = args[1] + listItem = args[2] + + content = loadYaml(filename) + appendItem(content, key, listItem) + writeYaml(filename, content) + + return 0 def removeKey(content, key): pieces = key.split(".", 1) @@ -69,6 +97,7 @@ def main(): commands = { "help": showUsage, + "append": append, "remove": remove, } From 9d62ade32e72f10e058428c1e441b0de225d8527 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 6 Feb 2024 11:14:27 -0500 Subject: [PATCH 18/25] update so-yaml tests --- salt/manager/tools/sbin/so-yaml.py | 2 +- salt/manager/tools/sbin/so-yaml_test.py | 44 +++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/so-yaml.py b/salt/manager/tools/sbin/so-yaml.py index 572585df9..9f4bdbb98 100755 --- a/salt/manager/tools/sbin/so-yaml.py +++ b/salt/manager/tools/sbin/so-yaml.py @@ -45,7 +45,7 @@ def appendItem(content, key, listItem): try: content[key].append(listItem) except TypeError: - print("The contents key provided is likely not a list. No action was taken on the file.") + print("The key provided is likely not a list. No action was taken on the file.") return 1 def append(args): diff --git a/salt/manager/tools/sbin/so-yaml_test.py b/salt/manager/tools/sbin/so-yaml_test.py index 7d0ed1a8e..f70314ba3 100644 --- a/salt/manager/tools/sbin/so-yaml_test.py +++ b/salt/manager/tools/sbin/so-yaml_test.py @@ -105,3 +105,47 @@ class TestRemove(unittest.TestCase): self.assertEqual(actual, expected) sysmock.assert_called_once_with(1) self.assertIn(mock_stdout.getvalue(), "Missing filename or key arg\n") + + def test_append(self): + filename = "/tmp/so-yaml_test-remove.yaml" + file = open(filename, "w") + file.write("{key1: { child1: 123, child2: abc }, key2: false, key3: [a,b,c]}") + file.close() + + soyaml.append([filename, "key3", "d"]) + + file = open(filename, "r") + actual = file.read() + file.close() + expected = "key1:\n child1: 123\n child2: abc\nkey2: false\nkey3:\n- a\n- b\n- c\n- d\n" + self.assertEqual(actual, expected) + + def test_append_nested(self): + filename = "/tmp/so-yaml_test-remove.yaml" + file = open(filename, "w") + file.write("{key1: { child1: 123, child2: [a,b,c] }, key2: false, key3: [e,f,g]}") + file.close() + + soyaml.append([filename, "key1.child2", "d"]) + + file = open(filename, "r") + actual = file.read() + file.close() + + expected = "key1:\n child1: 123\n child2:\n - a\n - b\n - c\n - d\nkey2: false\nkey3:\n- e\n- f\n- g\n" + self.assertEqual(actual, expected) + + def test_append_nested_deep(self): + filename = "/tmp/so-yaml_test-remove.yaml" + file = open(filename, "w") + file.write("{key1: { child1: 123, child2: { deep1: 45, deep2: [a,b,c] } }, key2: false, key3: [e,f,g]}") + file.close() + + soyaml.append([filename, "key1.child2.deep2", "d"]) + + file = open(filename, "r") + actual = file.read() + file.close() + + expected = "key1:\n child1: 123\n child2:\n deep1: 45\n deep2:\n - a\n - b\n - c\n - d\nkey2: false\nkey3:\n- e\n- f\n- g\n" + self.assertEqual(actual, expected) From 7106095128e643a4e50891c4631a406c35479cc8 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Tue, 6 Feb 2024 15:39:23 -0500 Subject: [PATCH 19/25] FEATURE: Improve Correlate and Hunt actions on SOC Actions menu #12315 --- salt/soc/defaults.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index c1b9470c8..86bb57c49 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -9,7 +9,7 @@ soc: icon: fa-crosshairs target: links: - - '/#/hunt?q="{value|escape}" | groupby event.module* event.dataset' + - '/#/hunt?q="{value|escape}" | groupby event.module* | groupby -sankey event.module* event.dataset | groupby event.dataset | groupby source.ip source.port destination.ip destination.port | groupby network.protocol | groupby source_geo.organization_name source.geo.country_name | groupby destination_geo.organization_name destination.geo.country_name | groupby rule.name rule.category event.severity_label | groupby dns.query.name | groupby file.mime_type | groupby http.virtual_host http.uri | groupby notice.note notice.message notice.sub_message | groupby ssl.server_name | groupby source.ip host.hostname user.name event.action event.type process.executable process.pid' - name: actionAddToCase description: actionAddToCaseHelp icon: fa-briefcase @@ -23,13 +23,13 @@ soc: icon: fab fa-searchengin target: '' links: - - '/#/hunt?q=("{:log.id.fuid}" OR "{:log.id.uid}" OR "{:network.community_id}") | groupby event.module* event.dataset' - - '/#/hunt?q=("{:log.id.fuid}" OR "{:log.id.uid}") | groupby event.module* event.dataset' - - '/#/hunt?q=("{:log.id.fuid}" OR "{:network.community_id}") | groupby event.module* event.dataset' - - '/#/hunt?q=("{:log.id.uid}" OR "{:network.community_id}") | groupby event.module* event.dataset' - - '/#/hunt?q="{:log.id.fuid}" | groupby event.module* event.dataset' - - '/#/hunt?q="{:log.id.uid}" | groupby event.module* event.dataset' - - '/#/hunt?q="{:network.community_id}" | groupby event.module* event.dataset' + - '/#/hunt?q=("{:log.id.fuid}" OR "{:log.id.uid}" OR "{:network.community_id}") | groupby event.module* | groupby -sankey event.module* event.dataset | groupby event.dataset | groupby source.ip source.port destination.ip destination.port | groupby network.protocol | groupby source_geo.organization_name source.geo.country_name | groupby destination_geo.organization_name destination.geo.country_name | groupby rule.name rule.category event.severity_label | groupby dns.query.name | groupby file.mime_type | groupby http.virtual_host http.uri | groupby notice.note notice.message notice.sub_message | groupby ssl.server_name | groupby source.ip host.hostname user.name event.action event.type process.executable process.pid' + - '/#/hunt?q=("{:log.id.fuid}" OR "{:log.id.uid}") | groupby event.module* | groupby -sankey event.module* event.dataset | groupby event.dataset | groupby source.ip source.port destination.ip destination.port | groupby network.protocol | groupby source_geo.organization_name source.geo.country_name | groupby destination_geo.organization_name destination.geo.country_name | groupby rule.name rule.category event.severity_label | groupby dns.query.name | groupby file.mime_type | groupby http.virtual_host http.uri | groupby notice.note notice.message notice.sub_message | groupby ssl.server_name | groupby source.ip host.hostname user.name event.action event.type process.executable process.pid' + - '/#/hunt?q=("{:log.id.fuid}" OR "{:network.community_id}") | groupby event.module* | groupby -sankey event.module* event.dataset | groupby event.dataset | groupby source.ip source.port destination.ip destination.port | groupby network.protocol | groupby source_geo.organization_name source.geo.country_name | groupby destination_geo.organization_name destination.geo.country_name | groupby rule.name rule.category event.severity_label | groupby dns.query.name | groupby file.mime_type | groupby http.virtual_host http.uri | groupby notice.note notice.message notice.sub_message | groupby ssl.server_name | groupby source.ip host.hostname user.name event.action event.type process.executable process.pid' + - '/#/hunt?q=("{:log.id.uid}" OR "{:network.community_id}") | groupby event.module* | groupby -sankey event.module* event.dataset | groupby event.dataset | groupby source.ip source.port destination.ip destination.port | groupby network.protocol | groupby source_geo.organization_name source.geo.country_name | groupby destination_geo.organization_name destination.geo.country_name | groupby rule.name rule.category event.severity_label | groupby dns.query.name | groupby file.mime_type | groupby http.virtual_host http.uri | groupby notice.note notice.message notice.sub_message | groupby ssl.server_name | groupby source.ip host.hostname user.name event.action event.type process.executable process.pid' + - '/#/hunt?q="{:log.id.fuid}" | groupby event.module* | groupby -sankey event.module* event.dataset | groupby event.dataset | groupby source.ip source.port destination.ip destination.port | groupby network.protocol | groupby source_geo.organization_name source.geo.country_name | groupby destination_geo.organization_name destination.geo.country_name | groupby rule.name rule.category event.severity_label | groupby dns.query.name | groupby file.mime_type | groupby http.virtual_host http.uri | groupby notice.note notice.message notice.sub_message | groupby ssl.server_name | groupby source.ip host.hostname user.name event.action event.type process.executable process.pid' + - '/#/hunt?q="{:log.id.uid}" | groupby event.module* | groupby -sankey event.module* event.dataset | groupby event.dataset | groupby source.ip source.port destination.ip destination.port | groupby network.protocol | groupby source_geo.organization_name source.geo.country_name | groupby destination_geo.organization_name destination.geo.country_name | groupby rule.name rule.category event.severity_label | groupby dns.query.name | groupby file.mime_type | groupby http.virtual_host http.uri | groupby notice.note notice.message notice.sub_message | groupby ssl.server_name | groupby source.ip host.hostname user.name event.action event.type process.executable process.pid' + - '/#/hunt?q="{:network.community_id}" | groupby event.module* | groupby -sankey event.module* event.dataset | groupby event.dataset | groupby source.ip source.port destination.ip destination.port | groupby network.protocol | groupby source_geo.organization_name source.geo.country_name | groupby destination_geo.organization_name destination.geo.country_name | groupby rule.name rule.category event.severity_label | groupby dns.query.name | groupby file.mime_type | groupby http.virtual_host http.uri | groupby notice.note notice.message notice.sub_message | groupby ssl.server_name | groupby source.ip host.hostname user.name event.action event.type process.executable process.pid' - name: actionPcap description: actionPcapHelp icon: fa-stream From b3f61536670050996fd539444863aa95a203a2b9 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 6 Feb 2024 16:15:54 -0500 Subject: [PATCH 20/25] update so-yaml tests --- salt/manager/tools/sbin/so-yaml.py | 7 +++- salt/manager/tools/sbin/so-yaml_test.py | 52 +++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/salt/manager/tools/sbin/so-yaml.py b/salt/manager/tools/sbin/so-yaml.py index 9f4bdbb98..24d520891 100755 --- a/salt/manager/tools/sbin/so-yaml.py +++ b/salt/manager/tools/sbin/so-yaml.py @@ -44,8 +44,11 @@ def appendItem(content, key, listItem): else: try: content[key].append(listItem) - except TypeError: - print("The key provided is likely not a list. No action was taken on the file.") + except AttributeError: + print("The key provided is not a list. No action was taken on the file.") + return 1 + except KeyError: + print("The key provided does not exist. No action was taken on the file.") return 1 def append(args): diff --git a/salt/manager/tools/sbin/so-yaml_test.py b/salt/manager/tools/sbin/so-yaml_test.py index f70314ba3..59db43860 100644 --- a/salt/manager/tools/sbin/so-yaml_test.py +++ b/salt/manager/tools/sbin/so-yaml_test.py @@ -149,3 +149,55 @@ class TestRemove(unittest.TestCase): expected = "key1:\n child1: 123\n child2:\n deep1: 45\n deep2:\n - a\n - b\n - c\n - d\nkey2: false\nkey3:\n- e\n- f\n- g\n" self.assertEqual(actual, expected) + + def test_append_key_noexist(self): + filename = "/tmp/so-yaml_test-append.yaml" + file = open(filename, "w") + file.write("{key1: { child1: 123, child2: { deep1: 45, deep2: [a,b,c] } }, key2: false, key3: [e,f,g]}") + file.close() + + with patch('sys.exit', new=MagicMock()) as sysmock: + with patch('sys.stdout', new=StringIO()) as mock_stdout: + sys.argv = ["cmd", "append", filename, "key4", "h"] + soyaml.main() + sysmock.assert_called() + self.assertEqual(mock_stdout.getvalue(), "The key provided does not exist. No action was taken on the file.\n") + + def test_append_key_noexist_deep(self): + filename = "/tmp/so-yaml_test-append.yaml" + file = open(filename, "w") + file.write("{key1: { child1: 123, child2: { deep1: 45, deep2: [a,b,c] } }, key2: false, key3: [e,f,g]}") + file.close() + + with patch('sys.exit', new=MagicMock()) as sysmock: + with patch('sys.stdout', new=StringIO()) as mock_stdout: + sys.argv = ["cmd", "append", filename, "key1.child2.deep3", "h"] + soyaml.main() + sysmock.assert_called() + self.assertEqual(mock_stdout.getvalue(), "The key provided does not exist. No action was taken on the file.\n") + + def test_append_key_nonlist(self): + filename = "/tmp/so-yaml_test-append.yaml" + file = open(filename, "w") + file.write("{key1: { child1: 123, child2: { deep1: 45, deep2: [a,b,c] } }, key2: false, key3: [e,f,g]}") + file.close() + + with patch('sys.exit', new=MagicMock()) as sysmock: + with patch('sys.stdout', new=StringIO()) as mock_stdout: + sys.argv = ["cmd", "append", filename, "key1", "h"] + soyaml.main() + sysmock.assert_called() + self.assertEqual(mock_stdout.getvalue(), "The key provided is not a list. No action was taken on the file.\n") + + def test_append_key_nonlist_deep(self): + filename = "/tmp/so-yaml_test-append.yaml" + file = open(filename, "w") + file.write("{key1: { child1: 123, child2: { deep1: 45, deep2: [a,b,c] } }, key2: false, key3: [e,f,g]}") + file.close() + + with patch('sys.exit', new=MagicMock()) as sysmock: + with patch('sys.stdout', new=StringIO()) as mock_stdout: + sys.argv = ["cmd", "append", filename, "key1.child2.deep1", "h"] + soyaml.main() + sysmock.assert_called() + self.assertEqual(mock_stdout.getvalue(), "The key provided is not a list. No action was taken on the file.\n") From 24fd3ef8cc40a4b0f3e7083ae935171079de11f8 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 6 Feb 2024 16:22:13 -0500 Subject: [PATCH 21/25] uopdate error message --- salt/manager/tools/sbin/so-yaml.py | 2 +- salt/manager/tools/sbin/so-yaml_test.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/manager/tools/sbin/so-yaml.py b/salt/manager/tools/sbin/so-yaml.py index 24d520891..41cab0b23 100755 --- a/salt/manager/tools/sbin/so-yaml.py +++ b/salt/manager/tools/sbin/so-yaml.py @@ -45,7 +45,7 @@ def appendItem(content, key, listItem): try: content[key].append(listItem) except AttributeError: - print("The key provided is not a list. No action was taken on the file.") + print("The existing value for the given key is not a list. No action was taken on the file.") return 1 except KeyError: print("The key provided does not exist. No action was taken on the file.") diff --git a/salt/manager/tools/sbin/so-yaml_test.py b/salt/manager/tools/sbin/so-yaml_test.py index 59db43860..488877ea1 100644 --- a/salt/manager/tools/sbin/so-yaml_test.py +++ b/salt/manager/tools/sbin/so-yaml_test.py @@ -187,7 +187,7 @@ class TestRemove(unittest.TestCase): sys.argv = ["cmd", "append", filename, "key1", "h"] soyaml.main() sysmock.assert_called() - self.assertEqual(mock_stdout.getvalue(), "The key provided is not a list. No action was taken on the file.\n") + self.assertEqual(mock_stdout.getvalue(), "The existing value for the given key is not a list. No action was taken on the file.\n") def test_append_key_nonlist_deep(self): filename = "/tmp/so-yaml_test-append.yaml" @@ -200,4 +200,4 @@ class TestRemove(unittest.TestCase): sys.argv = ["cmd", "append", filename, "key1.child2.deep1", "h"] soyaml.main() sysmock.assert_called() - self.assertEqual(mock_stdout.getvalue(), "The key provided is not a list. No action was taken on the file.\n") + self.assertEqual(mock_stdout.getvalue(), "The existing value for the given key is not a list. No action was taken on the file.\n") From f97d0f2f3644b298ef14c313fd9e94c39baa9573 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 7 Feb 2024 09:25:56 -0500 Subject: [PATCH 22/25] add /opt/so/rules/ to files_roots --- files/salt/master/master | 1 + salt/manager/tools/sbin/soup | 10 ++++++++++ salt/suricata/config.sls | 4 +++- 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/files/salt/master/master b/files/salt/master/master index b93fa93de..e309a560b 100644 --- a/files/salt/master/master +++ b/files/salt/master/master @@ -41,6 +41,7 @@ file_roots: base: - /opt/so/saltstack/local/salt - /opt/so/saltstack/default/salt + - /opt/so/rules # The master_roots setting configures a master-only copy of the file_roots dictionary, diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index a250116d1..6f086469a 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -594,6 +594,16 @@ up_to_2.4.50() { touch /opt/so/saltstack/local/pillar/stig/adv_stig.sls touch /opt/so/saltstack/local/pillar/stig/soc_stig.sls + # the file_roots need to be update due to salt 3006.6 upgrade not allowing symlinks outside the file_roots + # put new so-yaml in place + echo "Updating so-yaml" + \cp -v "$UPDATE_DIR/salt/manager/tools/sbin/so-yaml.py" "$DEFAULT_SALT_DIR/salt/manager/tools/sbin/" + \cp -v "$UPDATE_DIR/salt/manager/tools/sbin/so-yaml.py" /usr/sbin/ + echo "Creating a backup of the salt-master config." + cp -v /etc/salt/master "/etc/salt/master.so-$INSTALLEDVERSION" + echo "Adding /opt/so/rules to file_roots using so-yaml" + so-yaml.py append /etc/salt/master file_roots.base /opt/so/rules + INSTALLEDVERSION=2.4.50 } diff --git a/salt/suricata/config.sls b/salt/suricata/config.sls index 8d5279349..4804565ce 100644 --- a/salt/suricata/config.sls +++ b/salt/suricata/config.sls @@ -84,10 +84,12 @@ suridatadir: - mode: 770 - makedirs: True +# salt:// would resolve to /opt/so/rules because of the defined file_roots and +# nids not existing under /opt/so/saltstack/local/salt or /opt/so/saltstack/default/salt surirulesync: file.recurse: - name: /opt/so/conf/suricata/rules/ - - source: salt://suricata/rules/ + - source: salt://nids/ - user: 940 - group: 940 - show_changes: False From e42e07b245d32dd2bdb1d54e807d478209506efa Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 7 Feb 2024 13:05:45 -0500 Subject: [PATCH 23/25] update salt mine after salt-master restarts --- salt/manager/tools/sbin/soup | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 6f086469a..b4647bdbc 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -947,9 +947,6 @@ main() { systemctl_func "stop" "$cron_service_name" - # update mine items prior to stopping salt-minion and salt-master - update_salt_mine - echo "Updating dockers to $NEWVERSION." if [[ $is_airgap -eq 0 ]]; then airgap_update_dockers @@ -1025,6 +1022,9 @@ main() { salt-call state.apply salt.minion -l info queue=True echo "" + # ensure the mine is updated and populated before highstates run, following the salt-master restart + update_salt_mine + enable_highstate echo "" From 6534f392a9334a2dd1f1b89b9961d116d5c44de7 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 7 Feb 2024 14:25:28 -0500 Subject: [PATCH 24/25] update backup filename --- salt/manager/tools/sbin/soup | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index b4647bdbc..5bade9891 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -600,8 +600,11 @@ up_to_2.4.50() { \cp -v "$UPDATE_DIR/salt/manager/tools/sbin/so-yaml.py" "$DEFAULT_SALT_DIR/salt/manager/tools/sbin/" \cp -v "$UPDATE_DIR/salt/manager/tools/sbin/so-yaml.py" /usr/sbin/ echo "Creating a backup of the salt-master config." - cp -v /etc/salt/master "/etc/salt/master.so-$INSTALLEDVERSION" - echo "Adding /opt/so/rules to file_roots using so-yaml" + # INSTALLEDVERSION is 2.4.40 at this point, but we want the backup to have the version + # so was at prior to starting upgrade. use POSTVERSION here since it doesnt change until + # post upgrade changes. POSTVERSION set to INSTALLEDVERSION at start of soup + cp -v /etc/salt/master "/etc/salt/master.so-$POSTVERSION.bak" + echo "Adding /opt/so/rules to file_roots in /etc/salt/master using so-yaml" so-yaml.py append /etc/salt/master file_roots.base /opt/so/rules INSTALLEDVERSION=2.4.50 From d3d2305f00d90a2e076a8d377fcd9e72ea97a229 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Wed, 7 Feb 2024 16:08:27 -0500 Subject: [PATCH 25/25] FEATURE: Add new dashboards for community_id and firewall auth #12323 --- salt/soc/defaults.yaml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 86bb57c49..31b6eb588 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1424,8 +1424,11 @@ soc: - name: Zeek Notice description: Zeek notice logs query: 'event.dataset:zeek.notice | groupby -sankey notice.note destination.ip | groupby notice.note | groupby notice.message | groupby notice.sub_message | groupby source.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name' - - name: Connections - description: Network connection metadata + - name: Connections and Metadata with community_id + description: Network connections that include community_id + query: '_exists_:network.community_id | groupby event.module* | groupby -sankey event.module* event.dataset | groupby event.dataset | groupby source.ip source.port destination.ip destination.port | groupby network.protocol | groupby source_geo.organization_name source.geo.country_name | groupby destination_geo.organization_name destination.geo.country_name | groupby rule.name rule.category event.severity_label | groupby dns.query.name | groupby http.virtual_host http.uri | groupby notice.note notice.message notice.sub_message | groupby source.ip host.hostname user.name event.action event.type process.executable process.pid' + - name: Connections seen by Zeek or Suricata + description: Network connections logged by Zeek or Suricata query: 'tags:conn | groupby source.ip | groupby destination.ip | groupby destination.port | groupby -sankey destination.port network.protocol | groupby network.protocol | groupby network.transport | groupby connection.history | groupby connection.state | groupby connection.state_description | groupby source.geo.country_name | groupby destination.geo.country_name | groupby client.ip_bytes | groupby server.ip_bytes | groupby client.oui' - name: DCE_RPC description: DCE_RPC (Distributed Computing Environment / Remote Procedure Calls) network metadata @@ -1562,6 +1565,9 @@ soc: - name: Firewall description: Firewall logs query: 'observer.type:firewall | groupby -sankey event.action observer.ingress.interface.name | groupby event.action | groupby observer.ingress.interface.name | groupby network.type | groupby network.transport | groupby source.ip | groupby destination.ip | groupby destination.port' + - name: Firewall Auth + description: Firewall authentication logs + query: 'observer.type:firewall AND event.category:authentication | groupby user.name | groupby -sankey user.name source.ip | groupby source.ip | table soc_timestamp user.name source.ip message' - name: VLAN description: VLAN (Virtual Local Area Network) tagged logs query: '* AND _exists_:network.vlan.id | groupby network.vlan.id | groupby source.ip | groupby -sankey source.ip destination.ip | groupby destination.ip | groupby destination.port | groupby event.dataset | groupby event.module | groupby observer.name | groupby source.geo.country_name | groupby destination.geo.country_name'