From 5b7b6e5fb876d7c333603007e5e602e4c8a163ec Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Wed, 8 May 2024 14:00:23 -0400 Subject: [PATCH 001/101] FEATURE: Add more fields to the SOC Dashboards URL for so-import-pcap #12972 --- salt/common/tools/sbin_jinja/so-import-pcap | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin_jinja/so-import-pcap b/salt/common/tools/sbin_jinja/so-import-pcap index 30d5d4fc4..d3886305e 100755 --- a/salt/common/tools/sbin_jinja/so-import-pcap +++ b/salt/common/tools/sbin_jinja/so-import-pcap @@ -248,7 +248,7 @@ fi START_OLDEST_SLASH=$(echo $START_OLDEST | sed -e 's/-/%2F/g') END_NEWEST_SLASH=$(echo $END_NEWEST | sed -e 's/-/%2F/g') if [[ $VALID_PCAPS_COUNT -gt 0 ]] || [[ $SKIPPED_PCAPS_COUNT -gt 0 ]]; then - URL="https://{{ URLBASE }}/#/dashboards?q=$HASH_FILTERS%20%7C%20groupby%20-sankey%20event.dataset%20event.category%2a%20%7C%20groupby%20-pie%20event.category%20%7C%20groupby%20-bar%20event.module%20%7C%20groupby%20event.dataset%20%7C%20groupby%20event.module%20%7C%20groupby%20event.category%20%7C%20groupby%20observer.name%20%7C%20groupby%20source.ip%20%7C%20groupby%20destination.ip%20%7C%20groupby%20destination.port&t=${START_OLDEST_SLASH}%2000%3A00%3A00%20AM%20-%20${END_NEWEST_SLASH}%2000%3A00%3A00%20AM&z=UTC" + URL="https://{{ URLBASE }}/#/dashboards?q=$HASH_FILTERS%20%7C%20groupby%20event.module*%20%7C%20groupby%20-sankey%20event.module*%20event.dataset%20%7C%20groupby%20event.dataset%20%7C%20groupby%20source.ip%20%7C%20groupby%20destination.ip%20%7C%20groupby%20destination.port%20%7C%20groupby%20network.protocol%20%7C%20groupby%20rule.name%20rule.category%20event.severity_label%20%7C%20groupby%20dns.query.name%20%7C%20groupby%20file.mime_type%20%7C%20groupby%20http.virtual_host%20http.uri%20%7C%20groupby%20notice.note%20notice.message%20notice.sub_message%20%7C%20groupby%20ssl.server_name%20%7C%20groupby%20source_geo.organization_name%20source.geo.country_name%20%7C%20groupby%20destination_geo.organization_name%20destination.geo.country_name&t=${START_OLDEST_SLASH}%2000%3A00%3A00%20AM%20-%20${END_NEWEST_SLASH}%2000%3A00%3A00%20AM&z=UTC" status "Import complete!" status From ad9fdf064b579ec2399da5755d20209461feff2a Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 8 May 2024 15:24:29 -0400 Subject: [PATCH 002/101] Update config.sls --- salt/soc/config.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/soc/config.sls b/salt/soc/config.sls index b440b07fc..a85032295 100644 --- a/salt/soc/config.sls +++ b/salt/soc/config.sls @@ -82,7 +82,7 @@ socmotd: crondetectionsruntime: cron.present: - - name: /usr/local/bin/so-detections-runtime-status cron + - name: /usr/sbin/so-detections-runtime-status cron - identifier: detections-runtime-status - user: root - minute: '*/10' From 01a68568a66e8549a170d5c86ecef60b0d47a2cd Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 8 May 2024 16:37:13 -0400 Subject: [PATCH 003/101] Use state --- salt/strelka/backend/enabled.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/strelka/backend/enabled.sls b/salt/strelka/backend/enabled.sls index 863115eda..5510ffb0f 100644 --- a/salt/strelka/backend/enabled.sls +++ b/salt/strelka/backend/enabled.sls @@ -43,7 +43,7 @@ strelka_backend: {% endif %} - restart_policy: on-failure - watch: - - file: /opt/so/conf/strelka/rules/compiled/* + - file: strelkasensorrules delete_so-strelka-backend_so-status.disabled: file.uncomment: From 3a99624eb8a06f09c3f07c1f5449762bbe910f4e Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 9 May 2024 10:03:02 -0400 Subject: [PATCH 004/101] seperate manager states for strelka --- salt/strelka/backend/enabled.sls | 2 +- salt/strelka/config.sls | 38 ++++++-------------------------- 2 files changed, 8 insertions(+), 32 deletions(-) diff --git a/salt/strelka/backend/enabled.sls b/salt/strelka/backend/enabled.sls index 5510ffb0f..a26905e1f 100644 --- a/salt/strelka/backend/enabled.sls +++ b/salt/strelka/backend/enabled.sls @@ -43,7 +43,7 @@ strelka_backend: {% endif %} - restart_policy: on-failure - watch: - - file: strelkasensorrules + - file: strelkasensorcompiledrules delete_so-strelka-backend_so-status.disabled: file.uncomment: diff --git a/salt/strelka/config.sls b/salt/strelka/config.sls index cd8fb2667..e8f211bc8 100644 --- a/salt/strelka/config.sls +++ b/salt/strelka/config.sls @@ -5,45 +5,21 @@ {% from 'allowed_states.map.jinja' import allowed_states %} {% if sls.split('.')[0] in allowed_states %} +{% from 'vars/globals.map.jinja' import GLOBALS %} + +{% if GLOBALS.is_manager %} +include: + - strelka.manager +{% endif %} # Strelka config -strelkaconfdir: - file.directory: - - name: /opt/so/conf/strelka/rules/compiled/ - - user: 939 - - group: 939 - - makedirs: True - -strelkacompileyara: - file.managed: - - name: /opt/so/conf/strelka/compile_yara.py - - source: salt://strelka/compile_yara/compile_yara.py - - user: 939 - - group: 939 - - makedirs: True - -strelkarulesdir: - file.directory: - - name: /opt/so/conf/strelka/rules - - user: 939 - - group: 939 - - makedirs: True - -{%- if grains.role in ['so-sensor', 'so-heavynode'] %} -strelkasensorrules: +strelkasensorcompiledrules: file.recurse: - name: /opt/so/conf/strelka/rules/compiled/ - source: salt://strelka/rules/compiled/ - user: 939 - group: 939 - clean: True -{%- endif %} - -strelkareposdir: - file.directory: - - name: /opt/so/conf/strelka/repos - - user: 939 - - group: 939 - makedirs: True strelkadatadir: From a74fee4cd08df30a2d43b45705d6fedf7a970679 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 9 May 2024 11:26:02 -0400 Subject: [PATCH 005/101] strelka compiled rules --- salt/strelka/compile_yara/compile_yara.py | 11 +++++---- salt/strelka/config.sls | 27 ++++++++++------------- salt/strelka/manager.sls | 13 ++--------- 3 files changed, 21 insertions(+), 30 deletions(-) diff --git a/salt/strelka/compile_yara/compile_yara.py b/salt/strelka/compile_yara/compile_yara.py index dc77980d2..b6fa95899 100644 --- a/salt/strelka/compile_yara/compile_yara.py +++ b/salt/strelka/compile_yara/compile_yara.py @@ -20,7 +20,7 @@ def check_syntax(rule_file): def compile_yara_rules(rules_dir): compiled_dir = os.path.join(rules_dir, "compiled") - compiled_rules_path = [ os.path.join(compiled_dir, "rules.compiled"), "/opt/so/saltstack/default/salt/strelka/rules/compiled/rules.compiled" ] + compiled_rules_path = "/opt/so/saltstack/local/salt/strelka/rules/compiled/rules.compiled" rule_files = glob.glob(os.path.join(rules_dir, '**/*.yar'), recursive=True) files_to_compile = {} removed_count = 0 @@ -57,9 +57,12 @@ def compile_yara_rules(rules_dir): # Compile all remaining valid rules into a single file if files_to_compile: compiled_rules = yara.compile(filepaths=files_to_compile) - for path in compiled_rules_path: - compiled_rules.save(path) - print(f"All remaining rules compiled and saved into {path}") + compiled_rules.save(compiled_rules_path) + print(f"All remaining rules compiled and saved into {compiled_rules_path}") + # Remove the rules.compiled if there aren't any files to be compiled + else: + if os.path.exists(compiled_rules_path): + os.remove(compiled_rules_path) # Print summary of compilation results print(f"Summary: {success_count} rules compiled successfully, {removed_count} rules removed due to errors.") diff --git a/salt/strelka/config.sls b/salt/strelka/config.sls index e8f211bc8..f03afa61b 100644 --- a/salt/strelka/config.sls +++ b/salt/strelka/config.sls @@ -34,7 +34,18 @@ strelkalogdir: - name: /nsm/strelka/log - user: 939 - group: 939 - - makedirs: True + +strelkagkredisdatadir: + file.directory: + - name: /nsm/strelka/gk-redis-data + - user: 939 + - group: 939 + +strelkacoordredisdatadir: + file.directory: + - name: /nsm/strelka/coord-redis-data + - user: 939 + - group: 939 strelka_sbin: file.recurse: @@ -44,20 +55,6 @@ strelka_sbin: - group: 939 - file_mode: 755 -strelkagkredisdatadir: - file.directory: - - name: /nsm/strelka/gk-redis-data - - user: 939 - - group: 939 - - makedirs: True - -strelkacoordredisdatadir: - file.directory: - - name: /nsm/strelka/coord-redis-data - - user: 939 - - group: 939 - - makedirs: True - {% else %} {{sls}}_state_not_allowed: diff --git a/salt/strelka/manager.sls b/salt/strelka/manager.sls index 1c56a18fd..108a12deb 100644 --- a/salt/strelka/manager.sls +++ b/salt/strelka/manager.sls @@ -7,9 +7,9 @@ {% if sls in allowed_states %} # Strelka config -strelkaconfdir: +strelkarulesdir: file.directory: - - name: /opt/so/conf/strelka/rules/compiled/ + - name: /opt/so/conf/strelka/rules - user: 939 - group: 939 - makedirs: True @@ -20,21 +20,12 @@ strelkacompileyara: - source: salt://strelka/compile_yara/compile_yara.py - user: 939 - group: 939 - - makedirs: True - -strelkarulesdir: - file.directory: - - name: /opt/so/conf/strelka/rules - - user: 939 - - group: 939 - - makedirs: True strelkareposdir: file.directory: - name: /opt/so/conf/strelka/repos - user: 939 - group: 939 - - makedirs: True {% else %} From c864fec70cc1d06e5558d7de9cd12ff32675f530 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 9 May 2024 11:53:50 -0400 Subject: [PATCH 006/101] allow strelka.manager to run on standalone --- salt/strelka/manager.sls | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/strelka/manager.sls b/salt/strelka/manager.sls index 108a12deb..6a4aea416 100644 --- a/salt/strelka/manager.sls +++ b/salt/strelka/manager.sls @@ -4,7 +4,8 @@ # Elastic License 2.0. {% from 'allowed_states.map.jinja' import allowed_states %} -{% if sls in allowed_states %} +{# if strelka.manager or strelka in allowed_states #} +{% if sls in allowed_states or sls.split('.')[0] in allowed_states %} # Strelka config strelkarulesdir: From 823ff7ce11009ee9e85b7fbaef81a617395f1914 Mon Sep 17 00:00:00 2001 From: Wes Date: Thu, 9 May 2024 17:03:13 +0000 Subject: [PATCH 007/101] Remove exclusions and repos --- salt/strelka/defaults.yaml | 22 ---------------------- salt/strelka/rules/repos.txt.jinja | 2 -- salt/strelka/soc_strelka.yaml | 12 ------------ 3 files changed, 36 deletions(-) delete mode 100644 salt/strelka/rules/repos.txt.jinja diff --git a/salt/strelka/defaults.yaml b/salt/strelka/defaults.yaml index f91ad8691..4d69bf53b 100644 --- a/salt/strelka/defaults.yaml +++ b/salt/strelka/defaults.yaml @@ -733,28 +733,6 @@ strelka: enabled: False rules: enabled: True - repos: - - https://github.com/Security-Onion-Solutions/securityonion-yara.git - excluded: - - apt_flame2_orchestrator.yar - - apt_tetris.yar - - gen_susp_js_obfuscatorio.yar - - gen_webshells.yar - - generic_anomalies.yar - - general_cloaking.yar - - thor_inverse_matches.yar - - yara_mixed_ext_vars.yar - - apt_apt27_hyperbro.yar - - apt_turla_gazer.yar - - gen_google_anomaly.yar - - gen_icon_anomalies.yar - - gen_nvidia_leaked_cert.yar - - gen_sign_anomalies.yar - - gen_susp_xor.yar - - gen_webshells_ext_vars.yar - - configured_vulns_ext_vars.yar - - expl_outlook_cve_2023_23397.yar - - gen_mal_3cx_compromise_mar23.yar filecheck: historypath: '/nsm/strelka/history/' strelkapath: '/nsm/strelka/unprocessed/' diff --git a/salt/strelka/rules/repos.txt.jinja b/salt/strelka/rules/repos.txt.jinja deleted file mode 100644 index 043a02203..000000000 --- a/salt/strelka/rules/repos.txt.jinja +++ /dev/null @@ -1,2 +0,0 @@ -# DO NOT EDIT THIS FILE! Strelka YARA rule repos are stored here from the strelka:rules:repos pillar section -{{ STRELKAREPOS | join('\n') }} diff --git a/salt/strelka/soc_strelka.yaml b/salt/strelka/soc_strelka.yaml index e5240b9c9..947215bd5 100644 --- a/salt/strelka/soc_strelka.yaml +++ b/salt/strelka/soc_strelka.yaml @@ -578,18 +578,6 @@ strelka: global: False helpLink: strelka.html advanced: False - repos: - description: List of repos for so-yara-download to use to download rules. - readonly: False - global: False - helpLink: strelka.html - advanced: False - excluded: - description: List of rules to exclude so-yara-update from download and propagating to backend nodes. - readonly: False - global: False - helpLink: strelka.html - advanced: False filecheck: historypath: description: The path for previously scanned files. From 8a34f5621cfa7b881b6bb47ce8873d1a32c23d70 Mon Sep 17 00:00:00 2001 From: Wes Date: Thu, 9 May 2024 17:26:45 +0000 Subject: [PATCH 008/101] Remove old YARA download script --- .../manager/tools/sbin_jinja/so-yara-download | 51 ------------------- 1 file changed, 51 deletions(-) delete mode 100644 salt/manager/tools/sbin_jinja/so-yara-download diff --git a/salt/manager/tools/sbin_jinja/so-yara-download b/salt/manager/tools/sbin_jinja/so-yara-download deleted file mode 100644 index aa9576253..000000000 --- a/salt/manager/tools/sbin_jinja/so-yara-download +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/bash -NOROOT=1 -. /usr/sbin/so-common - -{%- set proxy = salt['pillar.get']('manager:proxy') %} -{%- set noproxy = salt['pillar.get']('manager:no_proxy', '') %} - -# Download the rules from the internet -{%- if proxy %} -export http_proxy={{ proxy }} -export https_proxy={{ proxy }} -export no_proxy="{{ noproxy }}" -{%- endif %} - -repos="/opt/so/conf/strelka/repos.txt" -output_dir=/nsm/rules/yara -gh_status=$(curl -s -o /dev/null -w "%{http_code}" https://github.com) -clone_dir="/tmp" -if [ "$gh_status" == "200" ] || [ "$gh_status" == "301" ]; then - - while IFS= read -r repo; do - if ! $(echo "$repo" | grep -qE '^#'); then - # Remove old repo if existing bc of previous error condition or unexpected disruption - repo_name=`echo $repo | awk -F '/' '{print $NF}'` - [ -d $output_dir/$repo_name ] && rm -rf $output_dir/$repo_name - - # Clone repo and make appropriate directories for rules - git clone $repo $clone_dir/$repo_name - echo "Analyzing rules from $clone_dir/$repo_name..." - mkdir -p $output_dir/$repo_name - # Ensure a copy of the license is available for the rules - [ -f $clone_dir/$repo_name/LICENSE ] && cp $clone_dir/$repo_name/LICENSE $output_dir/$repo_name - - # Copy over rules - for i in $(find $clone_dir/$repo_name -name "*.yar*"); do - rule_name=$(echo $i | awk -F '/' '{print $NF}') - cp $i $output_dir/$repo_name - done - rm -rf $clone_dir/$repo_name - fi - done < $repos - - echo "Done!" - -/usr/sbin/so-yara-update - -else - echo "Server returned $gh_status status code." - echo "No connectivity to Github...exiting..." - exit 1 -fi From ea4cf4291310ca489f3598dccd0a0750eb584050 Mon Sep 17 00:00:00 2001 From: Wes Date: Thu, 9 May 2024 17:26:54 +0000 Subject: [PATCH 009/101] Remove old YARA update script --- salt/manager/tools/sbin_jinja/so-yara-update | 41 -------------------- 1 file changed, 41 deletions(-) delete mode 100644 salt/manager/tools/sbin_jinja/so-yara-update diff --git a/salt/manager/tools/sbin_jinja/so-yara-update b/salt/manager/tools/sbin_jinja/so-yara-update deleted file mode 100644 index 07c940f47..000000000 --- a/salt/manager/tools/sbin_jinja/so-yara-update +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -NOROOT=1 -. /usr/sbin/so-common - -echo "Starting to check for yara rule updates at $(date)..." - -newcounter=0 -excludedcounter=0 -excluded_rules=({{ EXCLUDEDRULES | join(' ') }}) - -# Pull down the SO Rules -SORULEDIR=/nsm/rules/yara -OUTPUTDIR=/opt/so/saltstack/local/salt/strelka/rules - -mkdir -p $OUTPUTDIR -# remove all rules prior to copy so we can clear out old rules -rm -f $OUTPUTDIR/* - -for i in $(find $SORULEDIR -name "*.yar" -o -name "*.yara"); do - rule_name=$(echo $i | awk -F '/' '{print $NF}') - if [[ ! "${excluded_rules[*]}" =~ ${rule_name} ]]; then - echo "Adding rule: $rule_name..." - cp $i $OUTPUTDIR/$rule_name - ((newcounter++)) - else - echo "Excluding rule: $rule_name..." - ((excludedcounter++)) - fi -done - -if [ "$newcounter" -gt 0 ] || [ "$excludedcounter" -gt 0 ];then - echo "$newcounter rules added." - echo "$excludedcounter rule(s) excluded." -fi - -echo "Finished rule updates at $(date)..." From 6ed82d7b293f74f16041a73c63afceedcf1b2701 Mon Sep 17 00:00:00 2001 From: Wes Date: Thu, 9 May 2024 17:27:46 +0000 Subject: [PATCH 010/101] Remove YARA download in setup --- setup/so-setup | 6 ------ 1 file changed, 6 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index b76f9bb98..1b91318b4 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -761,16 +761,10 @@ if ! [[ -f $install_opt_file ]]; then if [[ ! $is_airgap ]]; then title "Downloading IDS Rules" logCmd "so-rule-update" - title "Downloading YARA rules" - logCmd "su socore -c '/usr/sbin/so-yara-download'" if [[ $monints || $is_import ]]; then title "Restarting Suricata to pick up the new rules" logCmd "so-suricata-restart" fi - if [[ $monints ]]; then - title "Restarting Strelka to use new rules" - logCmd "so-strelka-restart" - fi fi title "Setting up Kibana Default Space" logCmd "so-kibana-space-defaults" From 074d063feedd83786fa694ba5608f14646ef10d3 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 9 May 2024 14:52:58 -0400 Subject: [PATCH 011/101] tests will retry on any rule import failure --- setup/so-functions | 23 +++++++++++++++++++---- setup/so-setup | 3 +++ 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 80ad0be6a..3cdaee9ca 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1259,12 +1259,27 @@ soc_pillar() { " srvKey: '$SOCSRVKEY'"\ " modules:"\ " elastalertengine:"\ - " allowRegex: '$ELASTALERT_ALLOW_REGEX'"\ + " allowRegex: '$ELASTALERT_ALLOW_REGEX'" > "$soc_pillar_file" + if [[ -n "$ELASTALERT_FAIL_ERROR_COUNT" ]]; then + printf '%s\n'\ + " failAfterConsecutiveErrorCount: $ELASTALERT_FAIL_ERROR_COUNT" >> "$soc_pillar_file" + fi + + printf '%s\n'\ " strelkaengine:"\ - " allowRegex: '$STRELKA_ALLOW_REGEX'"\ + " allowRegex: '$STRELKA_ALLOW_REGEX'" >> "$soc_pillar_file" + if [[ -n "$STRELKA_FAIL_ERROR_COUNT" ]]; then + printf '%s\n'\ + " failAfterConsecutiveErrorCount: $STRELKA_FAIL_ERROR_COUNT" >> "$soc_pillar_file" + fi + + printf '%s\n'\ " suricataengine:"\ - " allowRegex: '$SURICATA_ALLOW_REGEX'"\ - "" > "$soc_pillar_file" + " allowRegex: '$SURICATA_ALLOW_REGEX'" >> "$soc_pillar_file" + if [[ -n "$SURICATA_FAIL_ERROR_COUNT" ]]; then + printf '%s\n'\ + " failAfterConsecutiveErrorCount: $SURICATA_FAIL_ERROR_COUNT" >> "$soc_pillar_file" + fi if [[ $telemetry -ne 0 ]]; then echo " telemetryEnabled: false" >> $soc_pillar_file diff --git a/setup/so-setup b/setup/so-setup index b76f9bb98..cb535469b 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -246,8 +246,11 @@ if [ -n "$test_profile" ]; then WEBPASSWD1=0n10nus3r WEBPASSWD2=0n10nus3r STRELKA_ALLOW_REGEX="EquationGroup_Toolset_Apr17__ELV_.*" + STRELKA_FAIL_ERROR_COUNT=1 ELASTALERT_ALLOW_REGEX="Security Onion" + ELASTALERT_FAIL_ERROR_COUNT=1 SURICATA_ALLOW_REGEX="(200033\\d|2100538|2102466)" + SURICATA_FAIL_ERROR_COUNT=1 update_sudoers_for_testing fi From 19e1aaa1a65358f6cebba8645c28d0604b79371c Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 9 May 2024 15:45:33 -0400 Subject: [PATCH 012/101] exclude detection rule errors --- salt/common/tools/sbin/so-log-check | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check index b83c98e7a..67eff6d54 100755 --- a/salt/common/tools/sbin/so-log-check +++ b/salt/common/tools/sbin/so-log-check @@ -201,6 +201,7 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Unknown column" # Elastalert errors from running EQL queries EXCLUDED_ERRORS="$EXCLUDED_ERRORS|parsing_exception" # Elastalert EQL parsing issue. Temp. EXCLUDED_ERRORS="$EXCLUDED_ERRORS|context deadline exceeded" + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Error running query:" # Specific issues with detection rules fi RESULT=0 From fecd674fdb878ed289308091da574792c06cc65c Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 9 May 2024 17:55:41 -0400 Subject: [PATCH 013/101] Add quick action to find related alerts for a detection --- salt/soc/defaults.yaml | 6 ++++++ salt/soc/merged.map.jinja | 1 + 2 files changed, 7 insertions(+) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index b6a52fd75..dcb84aea7 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -78,6 +78,12 @@ soc: target: '' links: - '/#/hunt?q=(process.entity_id:"{:process.entity_id}" OR process.entity_id:"{:process.Ext.ancestry|processAncestors}") | groupby event.dataset | groupby -sankey event.dataset event.action | groupby event.action | groupby process.parent.name | groupby -sankey process.parent.name process.name | groupby process.name | groupby process.command_line | groupby host.name user.name | groupby source.ip source.port destination.ip destination.port | groupby dns.question.name | groupby dns.answers.data | groupby file.path | groupby registry.path | groupby dll.path' + - name: actionRelatedAlerts + description: actionRelatedAlertsHelp + icon: fa-bell + links: + - '/#/alerts?q=rule.uuid: {:so_detection.publicId|escape} | groupby rule.name event.module* event.severity_label' + target: '' eventFields: default: - soc_timestamp diff --git a/salt/soc/merged.map.jinja b/salt/soc/merged.map.jinja index 222566dba..e53790dc1 100644 --- a/salt/soc/merged.map.jinja +++ b/salt/soc/merged.map.jinja @@ -79,6 +79,7 @@ {% do SOCMERGED.config.server.client.update({'job': {'actions': standard_actions}}) %} {% do SOCMERGED.config.server.client.alerts.update({'actions': standard_actions}) %} {% do SOCMERGED.config.server.client.cases.update({'actions': standard_actions}) %} +{% do SOCMERGED.config.server.client.detections.update({'actions': standard_actions}) %} {# replace the _x_ with . for soc ui to config conversion #} {% do SOCMERGED.config.eventFields.update({':endpoint:events.api': SOCMERGED.config.eventFields.pop(':endpoint:events_x_api') }) %} From a1291e43c368024bb53676a44640d1bad651ca6e Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Fri, 10 May 2024 07:58:13 -0400 Subject: [PATCH 014/101] FIX: so-index-list typo #12988 --- salt/elasticsearch/tools/sbin/so-index-list | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/tools/sbin/so-index-list b/salt/elasticsearch/tools/sbin/so-index-list index 1e4595b35..dbdd6fa29 100755 --- a/salt/elasticsearch/tools/sbin/so-index-list +++ b/salt/elasticsearch/tools/sbin/so-index-list @@ -7,4 +7,4 @@ -curl -K /opt/so/conf/elasticsearch/curl.config-X GET -k -L "https://localhost:9200/_cat/indices?v&s=index" +curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L "https://localhost:9200/_cat/indices?pretty&v&s=index" From 26cb8d43e13fc05cb0d2178e8cc82ce4f5338564 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Fri, 10 May 2024 08:01:56 -0400 Subject: [PATCH 015/101] FIX: so-index-list typo #12988 --- salt/elasticsearch/tools/sbin/so-index-list | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/tools/sbin/so-index-list b/salt/elasticsearch/tools/sbin/so-index-list index dbdd6fa29..572e55cba 100755 --- a/salt/elasticsearch/tools/sbin/so-index-list +++ b/salt/elasticsearch/tools/sbin/so-index-list @@ -5,6 +5,6 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. - +. /usr/sbin/so-common curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L "https://localhost:9200/_cat/indices?pretty&v&s=index" From 950c68783c9524cc49a0bc9f1477d6c851630a78 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 10 May 2024 11:46:00 -0400 Subject: [PATCH 016/101] add pkg policycoreutils-python-utils to idh node --- salt/idh/openssh/config.sls | 2 ++ salt/idh/openssh/init.sls | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/salt/idh/openssh/config.sls b/salt/idh/openssh/config.sls index d358bb5be..4bd177faa 100644 --- a/salt/idh/openssh/config.sls +++ b/salt/idh/openssh/config.sls @@ -11,6 +11,8 @@ idh_sshd_selinux: - sel_type: ssh_port_t - prereq: - file: openssh_config + - require: + - file: python_selinux_mgmt_tools {% endif %} openssh_config: diff --git a/salt/idh/openssh/init.sls b/salt/idh/openssh/init.sls index ba0a8ab04..79d082502 100644 --- a/salt/idh/openssh/init.sls +++ b/salt/idh/openssh/init.sls @@ -15,3 +15,9 @@ openssh: - enable: False - name: {{ openssh_map.service }} {% endif %} + +{% if grains.os_family == 'RedHat' %} +python_selinux_mgmt_tools: + pkg.installed: + - name: policycoreutils-python-utils +{% endif %} From 986cbb129a009fadd8044bc3dfa6706a81629ea7 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 10 May 2024 12:33:56 -0400 Subject: [PATCH 017/101] pkg not file --- salt/idh/openssh/config.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/idh/openssh/config.sls b/salt/idh/openssh/config.sls index 4bd177faa..5e2acd8d2 100644 --- a/salt/idh/openssh/config.sls +++ b/salt/idh/openssh/config.sls @@ -12,7 +12,7 @@ idh_sshd_selinux: - prereq: - file: openssh_config - require: - - file: python_selinux_mgmt_tools + - pkg: python_selinux_mgmt_tools {% endif %} openssh_config: From 2a0e33401df668f0ec71364d428f098fb2686225 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 10 May 2024 16:54:50 -0400 Subject: [PATCH 018/101] support upgrade tests --- salt/manager/tools/sbin/soup | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index fa3c3b5ee..c8acab0a6 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -584,9 +584,22 @@ up_to_2.4.60() { up_to_2.4.70() { playbook_migration toggle_telemetry + add_detection_test_pillars INSTALLEDVERSION=2.4.70 } +add_detection_test_pillars() { + if [[ -n "$SOUP_INTERNAL_TESTING" ]]; then + echo "Adding detection pillar values for automated testing" + so-yaml.py add /opt/so/saltstack/local/pillar/soc/soc_soc.sls soc.config.server.modules.elastalertengine.allowRegex SecurityOnion + so-yaml.py add /opt/so/saltstack/local/pillar/soc/soc_soc.sls soc.config.server.modules.elastalertengine.failAfterConsecutiveErrorCount 1 + so-yaml.py add /opt/so/saltstack/local/pillar/soc/soc_soc.sls soc.config.server.modules.strelkaengine.allowRegex "EquationGroup_Toolset_Apr17__ELV_.*" + so-yaml.py add /opt/so/saltstack/local/pillar/soc/soc_soc.sls soc.config.server.modules.strelkaengine.failAfterConsecutiveErrorCount 1 + so-yaml.py add /opt/so/saltstack/local/pillar/soc/soc_soc.sls soc.config.server.modules.suricataengine.allowRegex "(200033\\d|2100538|2102466)" + so-yaml.py add /opt/so/saltstack/local/pillar/soc/soc_soc.sls soc.config.server.modules.suricataengine.failAfterConsecutiveErrorCount 1 + fi +} + toggle_telemetry() { if [[ -z $UNATTENDED && $is_airgap -ne 0 ]]; then cat << ASSIST_EOF From 788c31014d5dac710981cf791352bebf000f4e7e Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Mon, 13 May 2024 08:30:48 -0400 Subject: [PATCH 019/101] Update README.md to reference new screenshots for 2.4.70 --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 19a560419..27502c787 100644 --- a/README.md +++ b/README.md @@ -8,19 +8,19 @@ Alerts ![Alerts](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/50_alerts.png) Dashboards -![Dashboards](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/51_dashboards.png) +![Dashboards](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/53_dashboards.png) Hunt -![Hunt](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/52_hunt.png) +![Hunt](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/56_hunt.png) PCAP -![PCAP](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/53_pcap.png) +![PCAP](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/62_pcap.png) Grid -![Grid](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/57_grid.png) +![Grid](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/75_grid.png) Config -![Config](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/61_config.png) +![Config](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/87_config.png) ### Release Notes From ae323cf38531ee3b29258c187dab94010abd5a12 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Mon, 13 May 2024 08:34:44 -0400 Subject: [PATCH 020/101] Update README.md to include new Detections screenshot --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 27502c787..a990326a8 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,9 @@ Dashboards Hunt ![Hunt](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/56_hunt.png) +Detections +![Detections](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/59_detections.png) + PCAP ![PCAP](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/62_pcap.png) From 641899ad562b4fc02ed182396f07459c284c3456 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 13 May 2024 09:50:14 -0400 Subject: [PATCH 021/101] Backup Suricata for migration and remove advanced from reverselookups --- salt/manager/tools/sbin/soup | 11 +++++++++++ salt/soc/soc_soc.yaml | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index aaa703ba9..f22bdec22 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -583,6 +583,7 @@ up_to_2.4.60() { up_to_2.4.70() { playbook_migration + suricata_idstools_migration toggle_telemetry add_detection_test_pillars INSTALLEDVERSION=2.4.70 @@ -634,6 +635,16 @@ ASSIST_EOF fi } +suricata_idstools_migration() { + #Backup the pillars for idstools + mkdir -p /nsm/backup/detections-migration/idstools + rsync -av /opt/so/saltstack/local/pillar/idstools /nsm/backup/detections-migration/idstools + + #Backup Thresholds + mkdir -p /nsm/backup/detections-migration/suricata + rsync -av /opt/so/saltstack/local/salt/suricata/thresholding /nsm/backup/detections-migration/suricata +} + playbook_migration() { # Start SOC Detections migration mkdir -p /nsm/backup/detections-migration/{suricata,sigma/rules,elastalert} diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index c908521fa..2a7659384 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -235,7 +235,7 @@ soc: apiTimeoutMs: description: Duration (in milliseconds) to wait for a response from the SOC server API before giving up and showing an error on the SOC UI. global: True - advanced: True + advanced: False forcedType: int webSocketTimeoutMs: description: Duration (in milliseconds) to wait for a response from the SOC server websocket before giving up and reconnecting. From 6c71c45ef6283d8c1849e706446168acc43a9dbc Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 13 May 2024 09:55:57 -0400 Subject: [PATCH 022/101] Update soup --- salt/manager/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index f22bdec22..b57af160d 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -638,7 +638,7 @@ ASSIST_EOF suricata_idstools_migration() { #Backup the pillars for idstools mkdir -p /nsm/backup/detections-migration/idstools - rsync -av /opt/so/saltstack/local/pillar/idstools /nsm/backup/detections-migration/idstools + rsync -av /opt/so/saltstack/local/pillar/idstools/* /nsm/backup/detections-migration/idstools #Backup Thresholds mkdir -p /nsm/backup/detections-migration/suricata From 28e40e42b388973ca3eca646458105ce970ab3cb Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 13 May 2024 09:58:32 -0400 Subject: [PATCH 023/101] Update soc_soc.yaml --- salt/soc/soc_soc.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index 2a7659384..c908521fa 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -235,7 +235,7 @@ soc: apiTimeoutMs: description: Duration (in milliseconds) to wait for a response from the SOC server API before giving up and showing an error on the SOC UI. global: True - advanced: False + advanced: True forcedType: int webSocketTimeoutMs: description: Duration (in milliseconds) to wait for a response from the SOC server websocket before giving up and reconnecting. From 9d6f6c7893f45f2d830f0d064278334b4e57bae6 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 13 May 2024 10:09:35 -0400 Subject: [PATCH 024/101] Update soup --- salt/manager/tools/sbin/soup | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index b57af160d..8e77fecf0 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -639,10 +639,20 @@ suricata_idstools_migration() { #Backup the pillars for idstools mkdir -p /nsm/backup/detections-migration/idstools rsync -av /opt/so/saltstack/local/pillar/idstools/* /nsm/backup/detections-migration/idstools + if [[ $? -eq 0 ]]; then + echo "IDStools configuration has been backed up." + else + fail "Error: rsync failed to copy the files. IDStools configuration has not been backed up." + fi #Backup Thresholds mkdir -p /nsm/backup/detections-migration/suricata rsync -av /opt/so/saltstack/local/salt/suricata/thresholding /nsm/backup/detections-migration/suricata + if [[ $? -eq 0 ]]; then + echo "Suricata thresholds have been backed up." + else + fail "Error: rsync failed to copy the files. Thresholds have not been backed up." + fi } playbook_migration() { From 649f52dac7a039f7f1fa05089d434c087755b026 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 13 May 2024 10:37:56 -0400 Subject: [PATCH 025/101] create_local_directories in soup too --- salt/common/tools/sbin/so-common | 15 +++++++++++++++ salt/manager/tools/sbin/soup | 2 ++ setup/so-functions | 15 --------------- setup/so-setup | 2 +- 4 files changed, 18 insertions(+), 16 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index a71d67f81..d6cd4c4e8 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -179,6 +179,21 @@ copy_new_files() { cd /tmp } +create_local_directories() { + info "Creating local pillar and salt directories if needed" + PILLARSALTDIR=$1 + for i in "pillar" "salt"; do + for d in $(find $PILLARSALTDIR/$i -type d); do + suffixdir=${d//$PILLARSALTDIR/} + if [ ! -d "$local_salt_dir/$suffixdir" ]; then + logCmd "mkdir -pv $local_salt_dir$suffixdir" + fi + done + logCmd "chown -R socore:socore $local_salt_dir/$i" + done + +} + disable_fastestmirror() { sed -i 's/enabled=1/enabled=0/' /etc/yum/pluginconf.d/fastestmirror.conf } diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index aaa703ba9..5a8f70771 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -1026,6 +1026,7 @@ main() { backup_old_states_pillars fi copy_new_files + create_local_directories "/opt/so/saltstack/default" apply_hotfix echo "Hotfix applied" update_version @@ -1092,6 +1093,7 @@ main() { echo "Copying new Security Onion code from $UPDATE_DIR to $DEFAULT_SALT_DIR." copy_new_files echo "" + create_local_directories "/opt/so/saltstack/default" update_version echo "" diff --git a/setup/so-functions b/setup/so-functions index 3cdaee9ca..d91161203 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -762,21 +762,6 @@ copy_salt_master_config() { logCmd "systemctl restart salt-master" } -create_local_directories() { - info "Creating local pillar and salt directories" - PILLARSALTDIR=${SCRIPTDIR::-5} - for i in "pillar" "salt"; do - for d in $(find $PILLARSALTDIR/$i -type d); do - suffixdir=${d//$PILLARSALTDIR/} - if [ ! -d "$local_salt_dir/$suffixdir" ]; then - logCmd "mkdir -pv $local_salt_dir$suffixdir" - fi - done - logCmd "chown -R socore:socore $local_salt_dir/$i" - done - -} - create_local_nids_rules() { title "Create a local.rules file so it doesn't get removed on updates" logCmd "mkdir -p /opt/so/saltstack/local/salt/idstools" diff --git a/setup/so-setup b/setup/so-setup index cb535469b..0387da0de 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -672,7 +672,7 @@ if ! [[ -f $install_opt_file ]]; then # Add the socore user add_socore_user_manager - create_local_directories + create_local_directories ${SCRIPTDIR::-5} setup_salt_master_dirs create_manager_pillars From 2643da978b282a8c55d7ae8cdd42e117caeeb859 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 13 May 2024 11:51:10 -0400 Subject: [PATCH 026/101] those functions in so-functions --- salt/common/tools/sbin/so-common | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index d6cd4c4e8..77e80ee8c 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -180,13 +180,13 @@ copy_new_files() { } create_local_directories() { - info "Creating local pillar and salt directories if needed" + echo "Creating local pillar and salt directories if needed" PILLARSALTDIR=$1 for i in "pillar" "salt"; do for d in $(find $PILLARSALTDIR/$i -type d); do suffixdir=${d//$PILLARSALTDIR/} if [ ! -d "$local_salt_dir/$suffixdir" ]; then - logCmd "mkdir -pv $local_salt_dir$suffixdir" + mkdir -pv $local_salt_dir$suffixdir fi done logCmd "chown -R socore:socore $local_salt_dir/$i" From eb038582308021c606490449aea8caf22a7300b7 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 13 May 2024 12:44:57 -0400 Subject: [PATCH 027/101] missed one --- salt/common/tools/sbin/so-common | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 77e80ee8c..0ed7a662d 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -189,9 +189,8 @@ create_local_directories() { mkdir -pv $local_salt_dir$suffixdir fi done - logCmd "chown -R socore:socore $local_salt_dir/$i" + chown -R socore:socore $local_salt_dir/$i done - } disable_fastestmirror() { From 26b5a3991252753226e8b64528d9e7ab6fe3354d Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 13 May 2024 12:59:17 -0400 Subject: [PATCH 028/101] Change index to detections.alerts --- salt/elastalert/files/modules/so/securityonion-es.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/elastalert/files/modules/so/securityonion-es.py b/salt/elastalert/files/modules/so/securityonion-es.py index 0a82bdce6..d9bb8009e 100644 --- a/salt/elastalert/files/modules/so/securityonion-es.py +++ b/salt/elastalert/files/modules/so/securityonion-es.py @@ -56,8 +56,8 @@ class SecurityOnionESAlerter(Alerter): "event_data": match, "@timestamp": timestamp } - url = f"https://{self.rule['es_host']}:{self.rule['es_port']}/logs-playbook.alerts-so/_doc/" + url = f"https://{self.rule['es_host']}:{self.rule['es_port']}/logs-detections.alerts-so/_doc/" requests.post(url, data=json.dumps(payload), headers=headers, verify=False, auth=creds) def get_info(self): - return {'type': 'SecurityOnionESAlerter'} \ No newline at end of file + return {'type': 'SecurityOnionESAlerter'} From c4c38f58cb145e2be9723c382cd8bf611eb2741e Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Mon, 13 May 2024 13:13:57 -0400 Subject: [PATCH 029/101] Update descriptions --- salt/idstools/soc_idstools.yaml | 2 +- salt/soc/soc_soc.yaml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/idstools/soc_idstools.yaml b/salt/idstools/soc_idstools.yaml index 698a7a1fc..993abfd51 100644 --- a/salt/idstools/soc_idstools.yaml +++ b/salt/idstools/soc_idstools.yaml @@ -9,7 +9,7 @@ idstools: forcedType: string helpLink: rules.html ruleset: - description: 'Defines the ruleset you want to run. Options are ETOPEN or ETPRO. Once you have changed the ruleset here, you will need to wait for the rule update to take place (every 8 hours), or you can force the update by nagivating to Detections --> Options dropdown menu --> Suricata --> Full Update. WARNING! Changing the ruleset will remove all existing Suricata rules of the previous ruleset and their associated overrides. This removal cannot be undone.' + description: 'Defines the ruleset you want to run. Options are ETOPEN or ETPRO. Once you have changed the ruleset here, you will need to wait for the rule update to take place (every 24 hours), or you can force the update by nagivating to Detections --> Options dropdown menu --> Suricata --> Full Update. WARNING! Changing the ruleset will remove all existing non-overlapping Suricata rules of the previous ruleset and their associated overrides. This removal cannot be undone.' global: True regex: ETPRO\b|ETOPEN\b helpLink: rules.html diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index c908521fa..fa8d80bc8 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -39,7 +39,7 @@ soc: helpLink: soc-customization.html sigma_final_pipeline__yaml: title: Final Sigma Pipeline - description: Final Processing Pipeline for Sigma Rules (future use, not yet complete) + description: Final Processing Pipeline for Sigma Rules. syntax: yaml file: True global: True @@ -115,7 +115,7 @@ soc: helpLink: sigma.html airgap: *eerulesRepos sigmaRulePackages: - description: 'Defines the Sigma Community Ruleset you want to run. One of these (core | core+ | core++ | all ) as well as an optional Add-on (emerging_threats_addon). Once you have changed the ruleset here, you will need to wait for the rule update to take place (every 8 hours), or you can force the update by nagivating to Detections --> Options dropdown menu --> Elastalert --> Full Update. WARNING! Changing the ruleset will remove all existing Sigma rules of the previous ruleset and their associated overrides. This removal cannot be undone.' + description: 'Defines the Sigma Community Ruleset you want to run. One of these (core | core+ | core++ | all ) as well as an optional Add-on (emerging_threats_addon). Once you have changed the ruleset here, you will need to wait for the scheduled rule update to take place (by default, every 24 hours), or you can force the update by nagivating to Detections --> Options dropdown menu --> Elastalert --> Full Update. WARNING! Changing the ruleset will remove all existing non-overlapping Sigma rules of the previous ruleset and their associated overrides. This removal cannot be undone.' global: True advanced: False helpLink: sigma.html @@ -255,7 +255,7 @@ soc: description: Set to true to enable case management in SOC. global: True detectionsEnabled: - description: Set to true to enable the Detections module in SOC. (future use, not yet complete) + description: Set to true to enable the Detections module in SOC. global: True inactiveTools: description: List of external tools to remove from the SOC UI. From e430de88d377d6fee6bc551cdc68e9f7b458ff56 Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Mon, 13 May 2024 13:15:06 -0400 Subject: [PATCH 030/101] Change rule updates to 24h --- salt/soc/defaults.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 8ef0047be..7b33adaa4 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1284,7 +1284,7 @@ soc: so-import: - securityonion-resources+critical - securityonion-resources+high - communityRulesImportFrequencySeconds: 28800 + communityRulesImportFrequencySeconds: 86400 communityRulesImportErrorSeconds: 300 failAfterConsecutiveErrorCount: 10 denyRegex: '' @@ -1353,7 +1353,7 @@ soc: autoEnabledYaraRules: - securityonion-yara autoUpdateEnabled: true - communityRulesImportFrequencySeconds: 28800 + communityRulesImportFrequencySeconds: 86400 communityRulesImportErrorSeconds: 300 failAfterConsecutiveErrorCount: 10 compileYaraPythonScriptPath: /opt/sensoroni/yara/compile_yara.py @@ -1373,7 +1373,7 @@ soc: suricataengine: allowRegex: '' autoUpdateEnabled: true - communityRulesImportFrequencySeconds: 28800 + communityRulesImportFrequencySeconds: 86400 communityRulesImportErrorSeconds: 300 failAfterConsecutiveErrorCount: 10 communityRulesFile: /nsm/rules/suricata/emerging-all.rules From c8870eae65dc395cf5f2165c4b8772dc41f8cca6 Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 13 May 2024 14:23:47 -0400 Subject: [PATCH 031/101] Add detection alerts template --- salt/elasticsearch/defaults.yaml | 62 ++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 156483b03..e54d58c3b 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -3591,6 +3591,68 @@ elasticsearch: set_priority: priority: 50 min_age: 30d + so-logs-detections_x_alerts: + index_sorting: false + index_template: + composed_of: + - so-data-streams-mappings + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + - so-logs-mappings + - so-logs-settings + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-detections.alerts-* + priority: 501 + template: + mappings: + _meta: + managed: true + managed_by: security_onion + package: + name: elastic_agent + settings: + index: + lifecycle: + name: so-logs-detections.alerts-so + mapping: + total_fields: + limit: 5001 + number_of_replicas: 0 + sort: + field: '@timestamp' + order: desc + policy: + _meta: + managed: true + managed_by: security_onion + package: + name: elastic_agent + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 1d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d so-logs-elastic_agent: index_sorting: false index_template: From d606f259d12b049d620d3f200085b91821995643 Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 13 May 2024 14:25:11 -0400 Subject: [PATCH 032/101] Add detection alerts --- salt/elasticsearch/soc_elasticsearch.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index cc92493fb..000fd60b7 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -394,6 +394,7 @@ elasticsearch: so-logs-darktrace_x_ai_analyst_alert: *indexSettings so-logs-darktrace_x_model_breach_alert: *indexSettings so-logs-darktrace_x_system_status_alert: *indexSettings + so-logs-detections_x_alerts: *indexSettings so-logs-f5_bigip_x_log: *indexSettings so-logs-fim_x_event: *indexSettings so-logs-fortinet_x_clientendpoint: *indexSettings From 1ef9509aac03a4b3911d9df343a249222056a61c Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 13 May 2024 14:34:22 -0400 Subject: [PATCH 033/101] define local_salt_dir --- salt/common/tools/sbin/so-common | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 0ed7a662d..8d4d9f8ab 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -182,6 +182,7 @@ copy_new_files() { create_local_directories() { echo "Creating local pillar and salt directories if needed" PILLARSALTDIR=$1 + local_salt_dir="/opt/so/saltstack/local" for i in "pillar" "salt"; do for d in $(find $PILLARSALTDIR/$i -type d); do suffixdir=${d//$PILLARSALTDIR/} From 13062099b3b5758c40182b00504c18489d88eee9 Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 13 May 2024 18:04:16 -0400 Subject: [PATCH 034/101] Remove YARA script update and reference to exclusions --- salt/manager/init.sls | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/salt/manager/init.sls b/salt/manager/init.sls index d979482ef..ec37f9ff3 100644 --- a/salt/manager/init.sls +++ b/salt/manager/init.sls @@ -73,17 +73,6 @@ manager_sbin: - exclude_pat: - "*_test.py" -yara_update_scripts: - file.recurse: - - name: /usr/sbin/ - - source: salt://manager/tools/sbin_jinja/ - - user: socore - - group: socore - - file_mode: 755 - - template: jinja - - defaults: - EXCLUDEDRULES: {{ STRELKAMERGED.rules.excluded }} - so-repo-file: file.managed: - name: /opt/so/conf/reposync/repodownload.conf From 5b45c80a62231b6f487e8b4a3f354435bca4f462 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Tue, 14 May 2024 10:01:18 -0400 Subject: [PATCH 035/101] FEATURE: Add NetFlow dashboard #13009 --- salt/soc/defaults.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 7b33adaa4..b96cabf9d 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1931,6 +1931,9 @@ soc: - name: ICS S7 description: S7 (Siemens) network metadata query: 'tags:s7* | groupby event.dataset | groupby -sankey event.dataset source.ip | groupby source.ip | groupby -sankey source.ip destination.ip | groupby destination.ip | groupby destination.port' + - name: NetFlow + description: NetFlow records + query: 'event.module:netflow | groupby source.ip | groupby -sankey source.ip destination.ip | groupby destination.ip | groupby destination.port | groupby network.type | groupby network.transport | groupby network.direction | groupby netflow.type | groupby netflow.exporter.version | groupby observer.ip | groupby source.as.organization.name | groupby source.geo.country_name | groupby destination.as.organization.name | groupby destination.geo.country_name' - name: Firewall description: Firewall logs query: 'observer.type:firewall | groupby event.action | groupby -sankey event.action observer.ingress.interface.name | groupby observer.ingress.interface.name | groupby network.type | groupby network.transport | groupby source.ip | groupby destination.ip | groupby destination.port' From 67645a662da34ee1931236a868549bdb2a1bc3ef Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Tue, 14 May 2024 10:14:16 -0400 Subject: [PATCH 036/101] FEATURE: Add NetFlow dashboard #13009 --- salt/soc/defaults.yaml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index b96cabf9d..ca64c6b7b 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1258,6 +1258,19 @@ soc: - event_data.destination.port - event_data.process.executable - event_data.process.pid + ':netflow:': + - soc_timestamp + - event.dataset + - source.ip + - source.port + - destination.ip + - destination.port + - network.type + - network.transport + - network.direction + - netflow.type + - netflow.exporter.version + - observer.ip server: bindAddress: 0.0.0.0:9822 baseUrl: / From 51862e580386f505ddd935a7cb08d9306a0864f7 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 14 May 2024 13:08:51 -0400 Subject: [PATCH 037/101] remove idh.services from idh node pillar files --- salt/manager/tools/sbin/so-minion | 6 +----- salt/manager/tools/sbin/soup | 8 +++++++- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index e0e892c3d..8b563ef1d 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -201,11 +201,7 @@ function add_idh_to_minion() { "idh:"\ " enabled: True"\ " restrict_management_ip: $IDH_MGTRESTRICT"\ - " services:" >> "$PILLARFILE" - IFS=',' read -ra IDH_SERVICES_ARRAY <<< "$IDH_SERVICES" - for service in ${IDH_SERVICES_ARRAY[@]}; do - echo " - $service" | tr '[:upper:]' '[:lower:]' | tr -d '"' >> "$PILLARFILE" - done + " " >> $PILLARFILE } function add_logstash_to_minion() { diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 39cd07071..0ac938188 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -438,7 +438,13 @@ post_to_2.4.60() { } post_to_2.4.70() { - echo "Nothing to apply" + echo "Removing idh.services from IDH node pillar files" + for file in /opt/so/saltstack/local/pillar/minions/*_idh.sls; do + if [[ ! $file =~ "/opt/so/saltstack/local/pillar/minions/adv_" ]]; then + echo "Removing idh.services from: $file" + so-yaml.py remove "$file" idh.services + fi + done POSTVERSION=2.4.70 } From 2dbbe8dec473e928d7f04ee7f537eb85b46b6799 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 15 May 2024 10:07:06 -0400 Subject: [PATCH 038/101] soup_scripts put so-yaml in salt file system. move soup scripts to manager.soup_scripts --- salt/{common => manager}/soup_scripts.sls | 16 ++++++++++++++++ salt/manager/tools/sbin/soup | 4 ++-- 2 files changed, 18 insertions(+), 2 deletions(-) rename salt/{common => manager}/soup_scripts.sls (75%) diff --git a/salt/common/soup_scripts.sls b/salt/manager/soup_scripts.sls similarity index 75% rename from salt/common/soup_scripts.sls rename to salt/manager/soup_scripts.sls index 90ee059a4..898de67ee 100644 --- a/salt/common/soup_scripts.sls +++ b/salt/manager/soup_scripts.sls @@ -1,3 +1,8 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + {% if '2.4' in salt['cp.get_file_str']('/etc/soversion') %} {% import_yaml '/opt/so/saltstack/local/pillar/global/soc_global.sls' as SOC_GLOBAL %} @@ -15,6 +20,8 @@ remove_common_so-firewall: file.absent: - name: /opt/so/saltstack/default/salt/common/tools/sbin/so-firewall +# This section is used to put the scripts in place in the Salt file system +# in case a state run tries to overwrite what we do in the next section. copy_so-common_common_tools_sbin: file.copy: - name: /opt/so/saltstack/default/salt/common/tools/sbin/so-common @@ -43,6 +50,15 @@ copy_so-firewall_manager_tools_sbin: - force: True - preserve: True +copy_so-yaml_manager_tools_sbin: + file.copy: + - name: /opt/so/saltstack/default/salt/manager/tools/sbin/so-yaml.py + - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-yaml.py + - force: True + - preserve: True + +# This section is used to put the new script in place so that it can be called during soup. +# It is faster than calling the states that normally manage them to put them in place. copy_so-common_sbin: file.copy: - name: /usr/sbin/so-common diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 0ac938188..24b6e5427 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -907,7 +907,7 @@ verify_latest_update_script() { else echo "You are not running the latest soup version. Updating soup and its components. This might take multiple runs to complete." - salt-call state.apply common.soup_scripts queue=True -lerror --file-root=$UPDATE_DIR/salt --local --out-file=/dev/null + salt-call state.apply manager.soup_scripts queue=True -lerror --file-root=$UPDATE_DIR/salt --local --out-file=/dev/null # Verify that soup scripts updated as expected get_soup_script_hashes @@ -915,7 +915,7 @@ verify_latest_update_script() { echo "Succesfully updated soup scripts." else echo "There was a problem updating soup scripts. Trying to rerun script update." - salt-call state.apply common.soup_scripts queue=True -linfo --file-root=$UPDATE_DIR/salt --local + salt-call state.apply manager.soup_scripts queue=True -linfo --file-root=$UPDATE_DIR/salt --local fi echo "" From 427b1e4524f3ec0267c5a54d58db48a523b60c9d Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 15 May 2024 10:28:02 -0400 Subject: [PATCH 039/101] revert soup_scripts back to common --- salt/{manager => common}/soup_scripts.sls | 0 salt/manager/tools/sbin/soup | 4 ++-- 2 files changed, 2 insertions(+), 2 deletions(-) rename salt/{manager => common}/soup_scripts.sls (100%) diff --git a/salt/manager/soup_scripts.sls b/salt/common/soup_scripts.sls similarity index 100% rename from salt/manager/soup_scripts.sls rename to salt/common/soup_scripts.sls diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 24b6e5427..0ac938188 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -907,7 +907,7 @@ verify_latest_update_script() { else echo "You are not running the latest soup version. Updating soup and its components. This might take multiple runs to complete." - salt-call state.apply manager.soup_scripts queue=True -lerror --file-root=$UPDATE_DIR/salt --local --out-file=/dev/null + salt-call state.apply common.soup_scripts queue=True -lerror --file-root=$UPDATE_DIR/salt --local --out-file=/dev/null # Verify that soup scripts updated as expected get_soup_script_hashes @@ -915,7 +915,7 @@ verify_latest_update_script() { echo "Succesfully updated soup scripts." else echo "There was a problem updating soup scripts. Trying to rerun script update." - salt-call state.apply manager.soup_scripts queue=True -linfo --file-root=$UPDATE_DIR/salt --local + salt-call state.apply common.soup_scripts queue=True -linfo --file-root=$UPDATE_DIR/salt --local fi echo "" From 7345d2c5a67d5d0437d3818e309b95ea87e4f9fb Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 15 May 2024 11:16:20 -0400 Subject: [PATCH 040/101] Update enabled.sls --- salt/soc/enabled.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/soc/enabled.sls b/salt/soc/enabled.sls index 6cea0c70d..38e1fd3fe 100644 --- a/salt/soc/enabled.sls +++ b/salt/soc/enabled.sls @@ -44,6 +44,7 @@ so-soc: - /opt/so/conf/soc/soc_users_roles:/opt/sensoroni/rbac/users_roles:rw - /opt/so/conf/soc/queue:/opt/sensoroni/queue:rw - /opt/so/saltstack:/opt/so/saltstack:rw + - /opt/so/conf/soc/migrations:/opt/so/conf/soc/migrations:rw - extra_hosts: {% for node in DOCKER_EXTRA_HOSTS %} {% for hostname, ip in node.items() %} From e3a08478672b9158ef652bf1c257a4a38e0f4d96 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 15 May 2024 11:31:41 -0400 Subject: [PATCH 041/101] Update soup --- salt/manager/tools/sbin/soup | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 0ac938188..edd7aec85 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -659,6 +659,10 @@ suricata_idstools_migration() { else fail "Error: rsync failed to copy the files. Thresholds have not been backed up." fi + + #Tell SOC to migrate + mkdir -p /opt/so/conf/soc/migrations + echo "0" > /opt/so/conf/soc/migrations/suricata-migration-2.4.70 } playbook_migration() { From ea253726a0ae6730e364067ebfa8f7cf854a18f0 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 15 May 2024 13:48:32 -0400 Subject: [PATCH 042/101] fix soup --- salt/manager/tools/sbin/soup | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 0ac938188..0e4f5f8c8 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -439,8 +439,8 @@ post_to_2.4.60() { post_to_2.4.70() { echo "Removing idh.services from IDH node pillar files" - for file in /opt/so/saltstack/local/pillar/minions/*_idh.sls; do - if [[ ! $file =~ "/opt/so/saltstack/local/pillar/minions/adv_" ]]; then + for file in /opt/so/saltstack/local/pillar/minions/*.sls; do + if [[ $file =~ "_idh.sls" && ! $file =~ "/opt/so/saltstack/local/pillar/minions/adv_" ]]; then echo "Removing idh.services from: $file" so-yaml.py remove "$file" idh.services fi From 8803ad401880a30ef108c70abdb9e8b536aa13df Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 15 May 2024 14:05:48 -0400 Subject: [PATCH 043/101] Update enabled.sls --- salt/soc/enabled.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/soc/enabled.sls b/salt/soc/enabled.sls index 38e1fd3fe..990bf210c 100644 --- a/salt/soc/enabled.sls +++ b/salt/soc/enabled.sls @@ -45,6 +45,7 @@ so-soc: - /opt/so/conf/soc/queue:/opt/sensoroni/queue:rw - /opt/so/saltstack:/opt/so/saltstack:rw - /opt/so/conf/soc/migrations:/opt/so/conf/soc/migrations:rw + - /nsm/backup/detections-migration:/nsm/backup/detections-migration:ro - extra_hosts: {% for node in DOCKER_EXTRA_HOSTS %} {% for hostname, ip in node.items() %} From 3a56058f7ff62c33d5c485d7a3934711fb632ef0 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 15 May 2024 15:31:31 -0400 Subject: [PATCH 044/101] update description --- salt/manager/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index df722e2ed..14d914df8 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -438,7 +438,7 @@ post_to_2.4.60() { } post_to_2.4.70() { - echo "Removing idh.services from IDH node pillar files" + echo "Removing idh.services from any existing IDH node pillar files" for file in /opt/so/saltstack/local/pillar/minions/*.sls; do if [[ $file =~ "_idh.sls" && ! $file =~ "/opt/so/saltstack/local/pillar/minions/adv_" ]]; then echo "Removing idh.services from: $file" From 6af030848246048f60b75c3b6f3216b184388402 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 15 May 2024 16:26:44 -0400 Subject: [PATCH 045/101] add a newline --- .../tools/sbin_jinja/so-elastic-agent-gen-installers | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers index ff46a3e07..1e4222cae 100755 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers @@ -72,5 +72,5 @@ do printf "\n### $GOOS/$GOARCH Installer Generated...\n" done -printf "\n### Cleaning up temp files in /nsm/elastic-agent-workspace" +printf "\n### Cleaning up temp files in /nsm/elastic-agent-workspace\n" rm -rf /nsm/elastic-agent-workspace From b4aec9a9d02543338959b79120cd18f941371db2 Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Wed, 15 May 2024 16:29:21 -0400 Subject: [PATCH 046/101] alphabetical order --- salt/soc/defaults.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index ca64c6b7b..1f9fe686b 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -2182,9 +2182,9 @@ soc: manualSync: customEnabled: false labels: - - Suricata - - Strelka - ElastAlert + - Strelka + - Suricata eventFields: default: - so_detection.title From 8076ea0e0aae1562f31b7d50c7a44f0665e53090 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 15 May 2024 16:34:05 -0400 Subject: [PATCH 047/101] add another space --- salt/manager/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 14d914df8..d9d8c298f 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -438,7 +438,7 @@ post_to_2.4.60() { } post_to_2.4.70() { - echo "Removing idh.services from any existing IDH node pillar files" + printf "\nRemoving idh.services from any existing IDH node pillar files\n" for file in /opt/so/saltstack/local/pillar/minions/*.sls; do if [[ $file =~ "_idh.sls" && ! $file =~ "/opt/so/saltstack/local/pillar/minions/adv_" ]]; then echo "Removing idh.services from: $file" From ab9ec2ec6b0002da76c671dfc8e2202aa64d01b2 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 15 May 2024 18:04:01 -0400 Subject: [PATCH 048/101] Update soup --- salt/manager/tools/sbin/soup | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index d9d8c298f..525fce3f6 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -663,6 +663,7 @@ suricata_idstools_migration() { #Tell SOC to migrate mkdir -p /opt/so/conf/soc/migrations echo "0" > /opt/so/conf/soc/migrations/suricata-migration-2.4.70 + chown -R socore:socore /opt/so/conf/soc/migrations } playbook_migration() { From 477181036112e7d8929e8cf0f5b4a835c2d4cad1 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 15 May 2024 19:10:50 -0400 Subject: [PATCH 049/101] exclude detect-parse errors --- salt/common/tools/sbin/so-log-check | 1 + salt/suricata/soc_suricata.yaml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check index 67eff6d54..5bee4d254 100755 --- a/salt/common/tools/sbin/so-log-check +++ b/salt/common/tools/sbin/so-log-check @@ -202,6 +202,7 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|parsing_exception" # Elastalert EQL parsing issue. Temp. EXCLUDED_ERRORS="$EXCLUDED_ERRORS|context deadline exceeded" EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Error running query:" # Specific issues with detection rules + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|detect-parse" # Suricata encountering a malformed rule fi RESULT=0 diff --git a/salt/suricata/soc_suricata.yaml b/salt/suricata/soc_suricata.yaml index 75ad1e476..e157ff852 100644 --- a/salt/suricata/soc_suricata.yaml +++ b/salt/suricata/soc_suricata.yaml @@ -12,7 +12,7 @@ suricata: title: SIDS helpLink: suricata.html readonlyUi: True - advanced: true + advanced: True classification: classification__config: description: Classifications config file. From 9796354b4841e4408b5bc349b3a643d025e72a8e Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 16 May 2024 14:27:32 -0400 Subject: [PATCH 050/101] dont merge policy from global_overrides if not defined in default index_settings --- salt/elasticsearch/template.map.jinja | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/salt/elasticsearch/template.map.jinja b/salt/elasticsearch/template.map.jinja index f5a124a9a..8d40d9e4d 100644 --- a/salt/elasticsearch/template.map.jinja +++ b/salt/elasticsearch/template.map.jinja @@ -19,6 +19,12 @@ {% set ES_INDEX_SETTINGS = {} %} {% do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.update(salt['defaults.merge'](ES_INDEX_SETTINGS_GLOBAL_OVERRIDES, ES_INDEX_PILLAR, in_place=False)) %} {% for index, settings in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.items() %} +{# if policy isn't defined in the original index settings, then dont merge policy from the global_overrides #} +{# this will prevent so-elasticsearch-ilm-policy-load from trying to load policy on non ILM manged indices #} +{% if not ES_INDEX_SETTINGS_ORIG[index].policy is defined %} +{% do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].pop('policy') %} +{% endif %} + {% if settings.index_template is defined %} {% if not settings.get('index_sorting', False) | to_bool and settings.index_template.template.settings.index.sort is defined %} {% do settings.index_template.template.settings.index.pop('sort') %} From 9d4668f4d32205ffabd7fc0d96708decdea6ff21 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Thu, 16 May 2024 15:45:55 -0400 Subject: [PATCH 051/101] Revert "dont merge policy from global_overrides if not defined in default index_settings" --- salt/elasticsearch/template.map.jinja | 6 ------ 1 file changed, 6 deletions(-) diff --git a/salt/elasticsearch/template.map.jinja b/salt/elasticsearch/template.map.jinja index 8d40d9e4d..f5a124a9a 100644 --- a/salt/elasticsearch/template.map.jinja +++ b/salt/elasticsearch/template.map.jinja @@ -19,12 +19,6 @@ {% set ES_INDEX_SETTINGS = {} %} {% do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.update(salt['defaults.merge'](ES_INDEX_SETTINGS_GLOBAL_OVERRIDES, ES_INDEX_PILLAR, in_place=False)) %} {% for index, settings in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.items() %} -{# if policy isn't defined in the original index settings, then dont merge policy from the global_overrides #} -{# this will prevent so-elasticsearch-ilm-policy-load from trying to load policy on non ILM manged indices #} -{% if not ES_INDEX_SETTINGS_ORIG[index].policy is defined %} -{% do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].pop('policy') %} -{% endif %} - {% if settings.index_template is defined %} {% if not settings.get('index_sorting', False) | to_bool and settings.index_template.template.settings.index.sort is defined %} {% do settings.index_template.template.settings.index.pop('sort') %} From b54632080ee4962985a2b50fd98bc5ae55c384fa Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 16 May 2024 16:04:17 -0400 Subject: [PATCH 052/101] check if exists in override before popping --- salt/elasticsearch/template.map.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/template.map.jinja b/salt/elasticsearch/template.map.jinja index 8d40d9e4d..4a90a4f54 100644 --- a/salt/elasticsearch/template.map.jinja +++ b/salt/elasticsearch/template.map.jinja @@ -21,7 +21,7 @@ {% for index, settings in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.items() %} {# if policy isn't defined in the original index settings, then dont merge policy from the global_overrides #} {# this will prevent so-elasticsearch-ilm-policy-load from trying to load policy on non ILM manged indices #} -{% if not ES_INDEX_SETTINGS_ORIG[index].policy is defined %} +{% if not ES_INDEX_SETTINGS_ORIG[index].policy is defined and ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy is defined %} {% do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].pop('policy') %} {% endif %} From cc6cb346e796569d5a4d0166ae43d3b43113e751 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 16 May 2024 16:31:45 -0400 Subject: [PATCH 053/101] fix issue/13030 --- salt/elasticsearch/template.map.jinja | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/salt/elasticsearch/template.map.jinja b/salt/elasticsearch/template.map.jinja index 4a90a4f54..b59c291a4 100644 --- a/salt/elasticsearch/template.map.jinja +++ b/salt/elasticsearch/template.map.jinja @@ -2,11 +2,9 @@ {% set DEFAULT_GLOBAL_OVERRIDES = ELASTICSEARCHDEFAULTS.elasticsearch.index_settings.pop('global_overrides') %} {% set PILLAR_GLOBAL_OVERRIDES = {} %} -{% if salt['pillar.get']('elasticsearch:index_settings') is defined %} -{% set ES_INDEX_PILLAR = salt['pillar.get']('elasticsearch:index_settings') %} -{% if ES_INDEX_PILLAR.global_overrides is defined %} -{% set PILLAR_GLOBAL_OVERRIDES = ES_INDEX_PILLAR.pop('global_overrides') %} -{% endif %} +{% set ES_INDEX_PILLAR = salt['pillar.get']('elasticsearch:index_settings', {}) %} +{% if ES_INDEX_PILLAR.global_overrides is defined %} +{% set PILLAR_GLOBAL_OVERRIDES = ES_INDEX_PILLAR.pop('global_overrides') %} {% endif %} {% set ES_INDEX_SETTINGS_ORIG = ELASTICSEARCHDEFAULTS.elasticsearch.index_settings %} From 34a5985311bd64ccb30aab66eac4eaf2a7694262 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Thu, 16 May 2024 21:14:57 -0400 Subject: [PATCH 054/101] Create tpm enrollment script Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/common/tools/sbin/so-luks-tpm-enroll | 60 +++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 salt/common/tools/sbin/so-luks-tpm-enroll diff --git a/salt/common/tools/sbin/so-luks-tpm-enroll b/salt/common/tools/sbin/so-luks-tpm-enroll new file mode 100644 index 000000000..f08fe5e0b --- /dev/null +++ b/salt/common/tools/sbin/so-luks-tpm-enroll @@ -0,0 +1,60 @@ +#!/bin/bash +# +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0." + +# This script is intended to be used in the case the ISO install did not properly setup TPM decrypt for LUKS partitions at boot. + +check_for_tpm() { + echo -n "Checking for TPM: " + if [ -d /sys/class/tpm/tpm0 ]; then + echo -e "tpm0 found." + TPM="yes" + # Check if TPM is using sha1 or sha256 + if [ -d /sys/class/tpm/tpm0/pcr-sha1 ]; then + echo -e "TPM is using sha1.\n" + TPM_PCR="sha1" + elif [ -d /sys/class/tpm/tpm0/pcr-sha256 ]; then + echo -e "TPM is using sha256.\n" + TPM_PCR="sha256" + fi + else + echo -e "No TPM found.\n" + exit 1 + fi +} + +check_for_luks_partitions() { + echo "Checking for LUKS partitions" + for part in $(lsblk -o NAME,FSTYPE -ln | grep crypto_LUKS | awk '{print $1}'); do + echo "Found LUKS partition: $part" + LUKS_PARTITIONS+=("$part") + done + if [ ${#LUKS_PARTITIONS[@]} -eq 0 ]; then + echo -e "No LUKS partitions found.\n" + exit 1 + fi + echo "" +} + +enroll_tpm_in_luks() { + read -s -p "Enter the LUKS passphrase used during ISO install: " LUKS_PASSPHRASE + echo "" + for part in "${LUKS_PARTITIONS[@]}"; do + echo "Enrolling TPM for LUKS device: /dev/$part" + if [ "$TPM_PCR" == "sha1" ]; then + clevis luks bind -d /dev/$part tpm2 '{"pcr_bank":"sha1","pcr_ids":"7"}' <<< $LUKS_PASSPHRASE + elif [ "$TPM_PCR" == "sha256" ]; then + clevis luks bind -d /dev/$part tpm2 '{"pcr_bank":"sha256","pcr_ids":"7"}' <<< $LUKS_PASSPHRASE + fi + done + echo "Running dracut" + dracut -fv --no-kernel +} + +check_for_tpm +check_for_luks_partitions +enroll_tpm_in_luks +echo -e "\nTPM enrollment complete. Reboot the system to verify the TPM is correctly decrypting the LUKS partition(s) at boot.\n" \ No newline at end of file From 1c4d36760afd9be9dcbdf47d9e1e62bc0edaef9d Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 17 May 2024 14:49:39 -0400 Subject: [PATCH 055/101] add support for custom alerters --- salt/elastalert/defaults.yaml | 1 + salt/elastalert/map.jinja | 5 +++++ salt/elastalert/soc_elastalert.yaml | 8 ++++++++ salt/soc/soc_soc.yaml | 7 +++++++ 4 files changed, 21 insertions(+) diff --git a/salt/elastalert/defaults.yaml b/salt/elastalert/defaults.yaml index a01c80952..393932992 100644 --- a/salt/elastalert/defaults.yaml +++ b/salt/elastalert/defaults.yaml @@ -1,5 +1,6 @@ elastalert: enabled: False + alerter_parameters: "" config: rules_folder: /opt/elastalert/rules/ scan_subdirectories: true diff --git a/salt/elastalert/map.jinja b/salt/elastalert/map.jinja index cc395d8ee..3db17d32b 100644 --- a/salt/elastalert/map.jinja +++ b/salt/elastalert/map.jinja @@ -13,3 +13,8 @@ {% do ELASTALERTDEFAULTS.elastalert.config.update({'es_password': pillar.elasticsearch.auth.users.so_elastic_user.pass}) %} {% set ELASTALERTMERGED = salt['pillar.get']('elastalert', ELASTALERTDEFAULTS.elastalert, merge=True) %} + +{% set params = ELASTALERTMERGED.alerter_parameters | load_yaml %} +{% if params != None %} + {% do ELASTALERTMERGED.config.update(params) %} +{% endif %} diff --git a/salt/elastalert/soc_elastalert.yaml b/salt/elastalert/soc_elastalert.yaml index cde09b83e..eec3f3866 100644 --- a/salt/elastalert/soc_elastalert.yaml +++ b/salt/elastalert/soc_elastalert.yaml @@ -2,6 +2,14 @@ elastalert: enabled: description: You can enable or disable Elastalert. helpLink: elastalert.html + alerter_parameters: + title: Alerter Parameters + description: Custom configuration parameters for additional, optional alerters that can be enabled for all Sigma rules. Filter for 'Additional Alerters' in this Configuration screen to find the setting that allows these alerters to be enabled within the SOC ElastAlert module. Use YAML format for these parameters, and reference the ElastAlert 2 documentation, located at https://elastalert2.readthedocs.io, for available alerters and their required configuration parameters. + global: True + multiline: True + syntax: yaml + helpLink: elastalert.html + forcedType: string config: disable_rules_on_error: description: Disable rules on failure. diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index fa8d80bc8..bc1c49185 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -83,6 +83,13 @@ soc: advanced: True modules: elastalertengine: + additionalAlerters: + title: Additional Alerters + description: Specify additional alerters to enable for all Sigma rules, one alerter name per line. Alerters refers to ElastAlert 2 alerters, as documented at https://elastalert2.readthedocs.io. Note that the configuration parameters for these alerters must be provided in the ElastAlert configuration section. Filter for 'Alerter Parameters' to find this related setting. + global: True + helpLink: sigma.html + forcedType: "[]string" + multiline: True allowRegex: description: 'Regex used to filter imported Sigma rules. Deny regex takes precedence over the Allow regex setting.' global: True From d9edff38df40ebe31618c311049a455f4feb8798 Mon Sep 17 00:00:00 2001 From: weslambert Date: Fri, 17 May 2024 16:10:10 -0400 Subject: [PATCH 056/101] Create compile report for SOC integrity check --- salt/strelka/compile_yara/compile_yara.py | 41 +++++++++++++++++++---- 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/salt/strelka/compile_yara/compile_yara.py b/salt/strelka/compile_yara/compile_yara.py index b6fa95899..cac60ad60 100644 --- a/salt/strelka/compile_yara/compile_yara.py +++ b/salt/strelka/compile_yara/compile_yara.py @@ -3,10 +3,13 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. +import argparse +import glob +import hashlib +import json import os import yara -import glob -import json +from datetime import datetime from concurrent.futures import ThreadPoolExecutor def check_syntax(rule_file): @@ -25,19 +28,24 @@ def compile_yara_rules(rules_dir): files_to_compile = {} removed_count = 0 success_count = 0 - + # Use ThreadPoolExecutor to parallelize syntax checks with ThreadPoolExecutor() as executor: results = executor.map(check_syntax, rule_files) - + # Collect yara files and prepare for batch compilation + ts = str(datetime.utcnow().isoformat()) + failure_ids = [] + success_ids = [] for success, rule_file, error_message in results: + rule_id = os.path.splitext(os.path.basename(rule_file))[0] if success: files_to_compile[os.path.basename(rule_file)] = rule_file success_count += 1 + success_ids.append(rule_id) else: + failure_ids.append(rule_id) # Extract just the UUID from the rule file name - rule_id = os.path.splitext(os.path.basename(rule_file))[0] log_entry = { "event_module": "soc", "event_dataset": "soc.detections", @@ -55,16 +63,37 @@ def compile_yara_rules(rules_dir): removed_count += 1 # Compile all remaining valid rules into a single file + compiled_sha256="" if files_to_compile: compiled_rules = yara.compile(filepaths=files_to_compile) compiled_rules.save(compiled_rules_path) print(f"All remaining rules compiled and saved into {compiled_rules_path}") + # Hash file + with open(compiled_rules_path, 'rb') as hash_file: + compiled_sha256=hashlib.sha256(hash_file.read()).hexdigest() # Remove the rules.compiled if there aren't any files to be compiled else: if os.path.exists(compiled_rules_path): os.remove(compiled_rules_path) + # Create compilation report + compilation_report = { + "timestamp": ts, + "compiled_sha256": compiled_sha256, + "failure": failure_ids, + "success": success_ids + } + + # Write total + with open('/opt/sensoroni/logs/detections_yara_compilation-total.log', 'w+') as report_file: + json.dump(compilation_report, report_file) + # Print summary of compilation results print(f"Summary: {success_count} rules compiled successfully, {removed_count} rules removed due to errors.") -compile_yara_rules("/opt/sensoroni/yara/rules/") +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Compile YARA rules from the specified directory") + parser.add_argument("rules_dir", help="Directory containing YARA rules to compile") + args = parser.parse_args() + +compile_yara_rules(args.rules_dir) From 0cc57fc24092145839fbee701867ef3132f6add1 Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Fri, 17 May 2024 15:47:23 -0600 Subject: [PATCH 057/101] Change Compilation Report Path Move compilation report path to /opt/so/state and mount that foulder in SOC --- salt/soc/enabled.sls | 1 + salt/strelka/compile_yara/compile_yara.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/soc/enabled.sls b/salt/soc/enabled.sls index 990bf210c..4d4b5f6fd 100644 --- a/salt/soc/enabled.sls +++ b/salt/soc/enabled.sls @@ -46,6 +46,7 @@ so-soc: - /opt/so/saltstack:/opt/so/saltstack:rw - /opt/so/conf/soc/migrations:/opt/so/conf/soc/migrations:rw - /nsm/backup/detections-migration:/nsm/backup/detections-migration:ro + - /opt/so/state:/opt/so/state:rw - extra_hosts: {% for node in DOCKER_EXTRA_HOSTS %} {% for hostname, ip in node.items() %} diff --git a/salt/strelka/compile_yara/compile_yara.py b/salt/strelka/compile_yara/compile_yara.py index cac60ad60..09e3f4680 100644 --- a/salt/strelka/compile_yara/compile_yara.py +++ b/salt/strelka/compile_yara/compile_yara.py @@ -85,7 +85,7 @@ def compile_yara_rules(rules_dir): } # Write total - with open('/opt/sensoroni/logs/detections_yara_compilation-total.log', 'w+') as report_file: + with open('/opt/so/state/detections_yara_compilation-total.log', 'w+') as report_file: json.dump(compilation_report, report_file) # Print summary of compilation results From fcc72a4f4ec9d31843bd7e620223735cc649a222 Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Mon, 20 May 2024 11:23:25 -0600 Subject: [PATCH 058/101] Add Default IntegrityCheck Frequency Values --- salt/soc/defaults.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 1f9fe686b..0113f22cc 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1305,6 +1305,7 @@ soc: reposFolder: /opt/sensoroni/sigma/repos rulesFingerprintFile: /opt/sensoroni/fingerprints/sigma.fingerprint stateFilePath: /opt/sensoroni/fingerprints/elastalertengine.state + integrityCheckFrequencySeconds: 600 rulesRepos: default: - repo: https://github.com/Security-Onion-Solutions/securityonion-resources @@ -1383,6 +1384,7 @@ soc: community: true yaraRulesFolder: /opt/sensoroni/yara/rules stateFilePath: /opt/sensoroni/fingerprints/strelkaengine.state + integrityCheckFrequencySeconds: 600 suricataengine: allowRegex: '' autoUpdateEnabled: true @@ -1393,6 +1395,7 @@ soc: denyRegex: '' rulesFingerprintFile: /opt/sensoroni/fingerprints/emerging-all.fingerprint stateFilePath: /opt/sensoroni/fingerprints/suricataengine.state + integrityCheckFrequencySeconds: 600 client: enableReverseLookup: false docsUrl: /docs/ From 6fac6eebceeacc74b6b595e9c7eedf26fa907305 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 20 May 2024 14:37:54 -0400 Subject: [PATCH 059/101] Helper script for enrolling tpm into luks Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- .../{so-luks-tpm-enroll => so-luks-tpm-regen} | 46 +++++++++++++++++-- 1 file changed, 42 insertions(+), 4 deletions(-) rename salt/common/tools/sbin/{so-luks-tpm-enroll => so-luks-tpm-regen} (67%) diff --git a/salt/common/tools/sbin/so-luks-tpm-enroll b/salt/common/tools/sbin/so-luks-tpm-regen similarity index 67% rename from salt/common/tools/sbin/so-luks-tpm-enroll rename to salt/common/tools/sbin/so-luks-tpm-regen index f08fe5e0b..50058b504 100644 --- a/salt/common/tools/sbin/so-luks-tpm-enroll +++ b/salt/common/tools/sbin/so-luks-tpm-regen @@ -5,7 +5,33 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0." +set -e # This script is intended to be used in the case the ISO install did not properly setup TPM decrypt for LUKS partitions at boot. +if [ -z $NOROOT ]; then + # Check for prerequisites + if [ "$(id -u)" -ne 0 ]; then + echo "This script must be run using sudo!" + exit 1 + fi +fi +ENROLL_TPM=N + +while [[ $# -gt 0 ]]; do + case $1 in + --enroll-tpm) + ENROLL_TPM=Y + ;; + *) + echo "Usage: $0 [options]" + echo "" + echo "where options are:" + echo " --enroll-tpm for when TPM enrollment was not selected during ISO install." + echo "" + exit 1 + ;; + esac + shift +done check_for_tpm() { echo -n "Checking for TPM: " @@ -50,11 +76,23 @@ enroll_tpm_in_luks() { clevis luks bind -d /dev/$part tpm2 '{"pcr_bank":"sha256","pcr_ids":"7"}' <<< $LUKS_PASSPHRASE fi done - echo "Running dracut" - dracut -fv --no-kernel + } + +regenerate_tpm_enrollment_token() { + for part in "${LUKS_PARTITIONS[@]}"; do + clevis luks regen -d /dev/$part -s 1 -q + done } check_for_tpm check_for_luks_partitions -enroll_tpm_in_luks -echo -e "\nTPM enrollment complete. Reboot the system to verify the TPM is correctly decrypting the LUKS partition(s) at boot.\n" \ No newline at end of file + +if [[ $ENROLL_TPM == "Y" ]]; then + enroll_tpm_in_luks +else + regenerate_tpm_enrollment_token +fi + +echo "Running dracut" +dracut -fv +echo -e "\nTPM configuration complete. Reboot the system to verify the TPM is correctly decrypting the LUKS partition(s) at boot.\n" \ No newline at end of file From 026023fd0a2c1643b5972d2198391c33972ba602 Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Mon, 20 May 2024 14:35:11 -0600 Subject: [PATCH 060/101] Annotate integrityCheckFrequencySeconds per det engine --- salt/soc/soc_soc.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index bc1c49185..1e6a915b0 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -113,6 +113,9 @@ soc: global: True advanced: True helpLink: sigma.html + integrityCheckFrequencySeconds: + description: 'How often the ElastAlert integrity checker runs (in seconds). This verifies the integrity of deployed rules.' + global: True rulesRepos: default: &eerulesRepos description: "Custom Git repos to pull Sigma rules from. 'license' field is required, 'folder' is optional. 'community' disables some management options for the imported rules - they can't be deleted or edited, just tuned, duplicated and Enabled | Disabled." @@ -211,6 +214,9 @@ soc: global: True advanced: True helpLink: yara.html + integrityCheckFrequencySeconds: + description: 'How often the Strelka integrity checker runs (in seconds). This verifies the integrity of deployed rules.' + global: True rulesRepos: default: &serulesRepos description: "Custom Git repos to pull YARA rules from. 'license' field is required, 'folder' is optional. 'community' disables some management options for the imported rules - they can't be deleted or edited, just tuned, duplicated and Enabled | Disabled." @@ -235,6 +241,9 @@ soc: global: True advanced: True helpLink: suricata.html + integrityCheckFrequencySeconds: + description: 'How often the Suricata integrity checker runs (in seconds). This verifies the integrity of deployed rules.' + global: True client: enableReverseLookup: description: Set to true to enable reverse DNS lookups for IP addresses in the SOC UI. From 6e97c39f5886c4e6816fd82c2472961f775c4a30 Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Mon, 20 May 2024 14:52:05 -0600 Subject: [PATCH 061/101] Marked as Advanced --- salt/soc/soc_soc.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index 1e6a915b0..0cbb99e62 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -116,6 +116,7 @@ soc: integrityCheckFrequencySeconds: description: 'How often the ElastAlert integrity checker runs (in seconds). This verifies the integrity of deployed rules.' global: True + advanced: True rulesRepos: default: &eerulesRepos description: "Custom Git repos to pull Sigma rules from. 'license' field is required, 'folder' is optional. 'community' disables some management options for the imported rules - they can't be deleted or edited, just tuned, duplicated and Enabled | Disabled." @@ -217,6 +218,7 @@ soc: integrityCheckFrequencySeconds: description: 'How often the Strelka integrity checker runs (in seconds). This verifies the integrity of deployed rules.' global: True + advanced: True rulesRepos: default: &serulesRepos description: "Custom Git repos to pull YARA rules from. 'license' field is required, 'folder' is optional. 'community' disables some management options for the imported rules - they can't be deleted or edited, just tuned, duplicated and Enabled | Disabled." @@ -244,6 +246,7 @@ soc: integrityCheckFrequencySeconds: description: 'How often the Suricata integrity checker runs (in seconds). This verifies the integrity of deployed rules.' global: True + advanced: True client: enableReverseLookup: description: Set to true to enable reverse DNS lookups for IP addresses in the SOC UI. From 6b2219b7f26d5244f9f0285c90400769761e31c2 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 20 May 2024 18:52:37 -0400 Subject: [PATCH 062/101] elastalert settings --- salt/elastalert/config.sls | 29 ++++++++++++++++ salt/elastalert/defaults.yaml | 2 +- salt/elastalert/enabled.sls | 2 ++ salt/elastalert/map.jinja | 21 ++++++++++-- salt/elastalert/soc_elastalert.yaml | 51 ++++++++++++++++++++++++++++- salt/soc/soc_soc.yaml | 2 +- salt/stig/soc_stig.yaml | 2 +- 7 files changed, 102 insertions(+), 7 deletions(-) diff --git a/salt/elastalert/config.sls b/salt/elastalert/config.sls index 252aa83c0..1251c9d19 100644 --- a/salt/elastalert/config.sls +++ b/salt/elastalert/config.sls @@ -82,6 +82,35 @@ elastasomodulesync: - group: 933 - makedirs: True +elastacustomdir: + file.directory: + - name: /opt/so/conf/elastalert/custom + - user: 933 + - group: 933 + - makedirs: True + +elastacustomsync: + file.recurse: + - name: /opt/so/conf/elastalert/custom + - source: salt://elastalert/files/custom + - user: 933 + - group: 933 + - makedirs: True + - template: jinja + - mode: 660 + - context: + elastalert: {{ ELASTALERTMERGED }} + - show_changes: False + +elastapredefinedsync: + file.recurse: + - name: /opt/so/conf/elastalert/predefined + - source: salt://elastalert/files/predefined + - user: 933 + - group: 933 + - makedirs: True + - show_changes: False + elastaconf: file.managed: - name: /opt/so/conf/elastalert/elastalert_config.yaml diff --git a/salt/elastalert/defaults.yaml b/salt/elastalert/defaults.yaml index 393932992..8021533ab 100644 --- a/salt/elastalert/defaults.yaml +++ b/salt/elastalert/defaults.yaml @@ -40,4 +40,4 @@ elastalert: level: INFO handlers: - file - propagate: false + propagate: False \ No newline at end of file diff --git a/salt/elastalert/enabled.sls b/salt/elastalert/enabled.sls index e4b3642db..6a1ff1440 100644 --- a/salt/elastalert/enabled.sls +++ b/salt/elastalert/enabled.sls @@ -30,6 +30,8 @@ so-elastalert: - /opt/so/rules/elastalert:/opt/elastalert/rules/:ro - /opt/so/log/elastalert:/var/log/elastalert:rw - /opt/so/conf/elastalert/modules/:/opt/elastalert/modules/:ro + - /opt/so/conf/elastalert/predefined/:/opt/elastalert/predefined/:ro + - /opt/so/conf/elastalert/custom/:/opt/elastalert/custom/:ro - /opt/so/conf/elastalert/elastalert_config.yaml:/opt/elastalert/config.yaml:ro {% if DOCKER.containers['so-elastalert'].custom_bind_mounts %} {% for BIND in DOCKER.containers['so-elastalert'].custom_bind_mounts %} diff --git a/salt/elastalert/map.jinja b/salt/elastalert/map.jinja index 3db17d32b..8d4e65652 100644 --- a/salt/elastalert/map.jinja +++ b/salt/elastalert/map.jinja @@ -14,7 +14,22 @@ {% set ELASTALERTMERGED = salt['pillar.get']('elastalert', ELASTALERTDEFAULTS.elastalert, merge=True) %} -{% set params = ELASTALERTMERGED.alerter_parameters | load_yaml %} -{% if params != None %} - {% do ELASTALERTMERGED.config.update(params) %} +{% if 'ntf' in salt['pillar.get']('features', []) %} + {% set params = ELASTALERTMERGED.alerter_parameters | load_yaml %} + {% if params != None %} + {% do ELASTALERTMERGED.config.update(params) %} + {% endif %} + + {% if ELASTALERTMERGED.smtp_user | length > 0 %} + {% do ELASTALERTMERGED.config.update({'smtp_auth_file': '/opt/elastalert/predefined/smtp_auth.yaml'}) %} + {% endif %} + + {% if ELASTALERTMERGED.smtp_user | length > 0 %} + {% do ELASTALERTMERGED.config.update({'smtp_auth_file': '/opt/elastalert/predefined/smtp_auth.yaml'}) %} + {% endif %} + + {% if ELASTALERTMERGED.jira_user | length > 0 or ELASTALERTMERGED.jira_key | length > 0 %} + {% do ELASTALERTMERGED.config.update({'jira_account_file': '/opt/elastalert/predefined/jira_auth.yaml'}) %} + {% endif %} + {% endif %} diff --git a/salt/elastalert/soc_elastalert.yaml b/salt/elastalert/soc_elastalert.yaml index eec3f3866..81df0541f 100644 --- a/salt/elastalert/soc_elastalert.yaml +++ b/salt/elastalert/soc_elastalert.yaml @@ -4,12 +4,61 @@ elastalert: helpLink: elastalert.html alerter_parameters: title: Alerter Parameters - description: Custom configuration parameters for additional, optional alerters that can be enabled for all Sigma rules. Filter for 'Additional Alerters' in this Configuration screen to find the setting that allows these alerters to be enabled within the SOC ElastAlert module. Use YAML format for these parameters, and reference the ElastAlert 2 documentation, located at https://elastalert2.readthedocs.io, for available alerters and their required configuration parameters. + description: Optional configuration parameters for additional alerters that can be enabled for all Sigma rules. Filter for 'Alerter' in this Configuration screen to find the setting that allows these alerters to be enabled within the SOC ElastAlert module. Use YAML format for these parameters, and reference the ElastAlert 2 documentation, located at https://elastalert2.readthedocs.io, for available alerters and their required configuration parameters. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key. global: True multiline: True syntax: yaml helpLink: elastalert.html forcedType: string + jira_api_key: + title: Jira API Key + description: Optional configuration parameter for Jira API Key, used instead of the Jira username and password. Requires a valid Security Onion license key. + global: True + sensitive: True + helpLink: elastalert.html + forcedType: string + jira_pass: + title: Jira Password + description: Optional configuration parameter for Jira password. Requires a valid Security Onion license key. + global: True + sensitive: True + helpLink: elastalert.html + forcedType: string + jira_user: + title: Jira Username + description: Optional configuration parameter for Jira username. Requires a valid Security Onion license key. + global: True + helpLink: elastalert.html + forcedType: string + smtp_pass: + title: SMTP Password + description: Optional configuration parameter for SMTP password, required for authenticating email servers. Requires a valid Security Onion license key. + global: True + sensitive: True + helpLink: elastalert.html + forcedType: string + smtp_user: + title: SMTP Username + description: Optional configuration parameter for SMTP username, required for authenticating email servers. Requires a valid Security Onion license key. + global: True + helpLink: elastalert.html + forcedType: string + opsgenie_key: + title: OpsGenie API Key + description: Optional configuration parameter for OpsGenie API Key. Requires a valid Security Onion license key. + global: True + sensitive: True + helpLink: elastalert.html + forcedType: string + files: + custom: + filename__ext: + title: Custom Parameter File + description: Optional configuration file that can be used to specify custom file contents, such as a SMTP certificate file. When used, the corresponding parameter must be set to this setting's filename.ext path inside the custom subdirectory. For example, if specifying the SMTP cert file, the smtp_cert_file key must be set to /opt/elastalert/custom/smtp.crt in the Alerter Parameters setting for this certificate to be enabled, and assumes this duplicated setting has been named smtp__crt. Note that double underscores will be replaced with a period in the filename. + global: True + duplicating: True + file: True + helpLink: elastalert.html config: disable_rules_on_error: description: Disable rules on failure. diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index bc1c49185..7367c030d 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -85,7 +85,7 @@ soc: elastalertengine: additionalAlerters: title: Additional Alerters - description: Specify additional alerters to enable for all Sigma rules, one alerter name per line. Alerters refers to ElastAlert 2 alerters, as documented at https://elastalert2.readthedocs.io. Note that the configuration parameters for these alerters must be provided in the ElastAlert configuration section. Filter for 'Alerter Parameters' to find this related setting. + description: Specify additional alerters to enable for all Sigma rules, one alerter name per line. Alerters refers to ElastAlert 2 alerters, as documented at https://elastalert2.readthedocs.io. Note that the configuration parameters for these alerters must be provided in the ElastAlert configuration section. Filter for 'Alerter' to find this related setting. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key. global: True helpLink: sigma.html forcedType: "[]string" diff --git a/salt/stig/soc_stig.yaml b/salt/stig/soc_stig.yaml index 1fb030c31..597aab809 100644 --- a/salt/stig/soc_stig.yaml +++ b/salt/stig/soc_stig.yaml @@ -1,6 +1,6 @@ stig: enabled: - description: You can enable or disable the application of STIGS using oscap. Note that the actions performed by OSCAP are not automatically reversible. + description: You can enable or disable the application of STIGS using oscap. Note that the actions performed by OSCAP are not automatically reversible. Requires a valid Security Onion license key. forcedType: bool advanced: True run_interval: From c594168b650f83fb015b7d6cb8dd6180ba225294 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 20 May 2024 19:05:43 -0400 Subject: [PATCH 063/101] elastalert settings --- salt/elastalert/config.sls | 3 ++- salt/elastalert/files/custom/placeholder | 1 + salt/elastalert/files/predefined/jira_auth.yaml | 6 ++++++ salt/elastalert/files/predefined/smtp_auth.yaml | 2 ++ 4 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 salt/elastalert/files/custom/placeholder create mode 100644 salt/elastalert/files/predefined/jira_auth.yaml create mode 100644 salt/elastalert/files/predefined/smtp_auth.yaml diff --git a/salt/elastalert/config.sls b/salt/elastalert/config.sls index 1251c9d19..0583cadfd 100644 --- a/salt/elastalert/config.sls +++ b/salt/elastalert/config.sls @@ -97,7 +97,7 @@ elastacustomsync: - group: 933 - makedirs: True - template: jinja - - mode: 660 + - file_mode: 660 - context: elastalert: {{ ELASTALERTMERGED }} - show_changes: False @@ -109,6 +109,7 @@ elastapredefinedsync: - user: 933 - group: 933 - makedirs: True + - file_mode: 660 - show_changes: False elastaconf: diff --git a/salt/elastalert/files/custom/placeholder b/salt/elastalert/files/custom/placeholder new file mode 100644 index 000000000..42e4ae4f0 --- /dev/null +++ b/salt/elastalert/files/custom/placeholder @@ -0,0 +1 @@ +THIS IS A PLACEHOLDER FILE \ No newline at end of file diff --git a/salt/elastalert/files/predefined/jira_auth.yaml b/salt/elastalert/files/predefined/jira_auth.yaml new file mode 100644 index 000000000..3f537c92b --- /dev/null +++ b/salt/elastalert/files/predefined/jira_auth.yaml @@ -0,0 +1,6 @@ +{% if elastalert.jira_user | length > 0 %} +user: {{ elastalert.jira_user }} +password: {{ elastalert.jira_pass }} +{% else %} +apikey: {{ elastalert.jira_key }} +{% endif %} \ No newline at end of file diff --git a/salt/elastalert/files/predefined/smtp_auth.yaml b/salt/elastalert/files/predefined/smtp_auth.yaml new file mode 100644 index 000000000..483dd6810 --- /dev/null +++ b/salt/elastalert/files/predefined/smtp_auth.yaml @@ -0,0 +1,2 @@ +user: {{ elastalert.smtp_user }} +password: {{ elastalert.smtp_pass }} \ No newline at end of file From 8a3061fe3e2369a61e7d6c3a7b12752ac9a51528 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 20 May 2024 19:36:06 -0400 Subject: [PATCH 064/101] elastalert settings --- .../files/predefined/{jira_auth.yaml => jira_auth.yaml.jinja} | 0 .../files/predefined/{smtp_auth.yaml => smtp_auth.yaml.jinja} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename salt/elastalert/files/predefined/{jira_auth.yaml => jira_auth.yaml.jinja} (100%) rename salt/elastalert/files/predefined/{smtp_auth.yaml => smtp_auth.yaml.jinja} (100%) diff --git a/salt/elastalert/files/predefined/jira_auth.yaml b/salt/elastalert/files/predefined/jira_auth.yaml.jinja similarity index 100% rename from salt/elastalert/files/predefined/jira_auth.yaml rename to salt/elastalert/files/predefined/jira_auth.yaml.jinja diff --git a/salt/elastalert/files/predefined/smtp_auth.yaml b/salt/elastalert/files/predefined/smtp_auth.yaml.jinja similarity index 100% rename from salt/elastalert/files/predefined/smtp_auth.yaml rename to salt/elastalert/files/predefined/smtp_auth.yaml.jinja From e2d0b8f4c7caa825de0ae494ebad01b5d2454f6d Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 20 May 2024 19:38:36 -0400 Subject: [PATCH 065/101] elastalert settings --- salt/elastalert/config.sls | 3 +++ 1 file changed, 3 insertions(+) diff --git a/salt/elastalert/config.sls b/salt/elastalert/config.sls index 0583cadfd..c435a212b 100644 --- a/salt/elastalert/config.sls +++ b/salt/elastalert/config.sls @@ -109,7 +109,10 @@ elastapredefinedsync: - user: 933 - group: 933 - makedirs: True + - template: jinja - file_mode: 660 + - context: + elastalert: {{ ELASTALERTMERGED }} - show_changes: False elastaconf: From f8ce039065e970dfce0a8fc47632b6395e9e1c65 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 20 May 2024 19:58:12 -0400 Subject: [PATCH 066/101] elastalert settings --- salt/elastalert/files/predefined/jira_auth.yaml.jinja | 4 ++-- salt/elastalert/files/predefined/smtp_auth.yaml.jinja | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/elastalert/files/predefined/jira_auth.yaml.jinja b/salt/elastalert/files/predefined/jira_auth.yaml.jinja index 3f537c92b..8ae240157 100644 --- a/salt/elastalert/files/predefined/jira_auth.yaml.jinja +++ b/salt/elastalert/files/predefined/jira_auth.yaml.jinja @@ -1,6 +1,6 @@ -{% if elastalert.jira_user | length > 0 %} +{% if elastalert.get('jira_user', '') | length > 0 and elastalert.get('jira_pass', '') | length > 0 %} user: {{ elastalert.jira_user }} password: {{ elastalert.jira_pass }} {% else %} -apikey: {{ elastalert.jira_key }} +apikey: {{ elastalert.get('jira_key', '') }} {% endif %} \ No newline at end of file diff --git a/salt/elastalert/files/predefined/smtp_auth.yaml.jinja b/salt/elastalert/files/predefined/smtp_auth.yaml.jinja index 483dd6810..0d488dd5c 100644 --- a/salt/elastalert/files/predefined/smtp_auth.yaml.jinja +++ b/salt/elastalert/files/predefined/smtp_auth.yaml.jinja @@ -1,2 +1,2 @@ -user: {{ elastalert.smtp_user }} -password: {{ elastalert.smtp_pass }} \ No newline at end of file +user: {{ elastalert.get('smtp_user', '') }} +password: {{ elastalert.get('smtp_pass', '') }} \ No newline at end of file From 02b4d37c11e8f162c358bd02929de41d9ce7c24c Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 20 May 2024 20:00:31 -0400 Subject: [PATCH 067/101] elastalert settings --- .../files/predefined/{jira_auth.yaml.jinja => jira_auth.yaml} | 0 .../files/predefined/{smtp_auth.yaml.jinja => smtp_auth.yaml} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename salt/elastalert/files/predefined/{jira_auth.yaml.jinja => jira_auth.yaml} (100%) rename salt/elastalert/files/predefined/{smtp_auth.yaml.jinja => smtp_auth.yaml} (100%) diff --git a/salt/elastalert/files/predefined/jira_auth.yaml.jinja b/salt/elastalert/files/predefined/jira_auth.yaml similarity index 100% rename from salt/elastalert/files/predefined/jira_auth.yaml.jinja rename to salt/elastalert/files/predefined/jira_auth.yaml diff --git a/salt/elastalert/files/predefined/smtp_auth.yaml.jinja b/salt/elastalert/files/predefined/smtp_auth.yaml similarity index 100% rename from salt/elastalert/files/predefined/smtp_auth.yaml.jinja rename to salt/elastalert/files/predefined/smtp_auth.yaml From b7a4f20c61be693f893c92c008831192a6fce1d0 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 20 May 2024 20:11:30 -0400 Subject: [PATCH 068/101] elastalert settings --- salt/elastalert/files/predefined/jira_auth.yaml | 2 +- salt/elastalert/files/predefined/smtp_auth.yaml | 2 +- salt/elastalert/soc_elastalert.yaml | 9 +-------- 3 files changed, 3 insertions(+), 10 deletions(-) diff --git a/salt/elastalert/files/predefined/jira_auth.yaml b/salt/elastalert/files/predefined/jira_auth.yaml index 8ae240157..9bf0425c0 100644 --- a/salt/elastalert/files/predefined/jira_auth.yaml +++ b/salt/elastalert/files/predefined/jira_auth.yaml @@ -2,5 +2,5 @@ user: {{ elastalert.jira_user }} password: {{ elastalert.jira_pass }} {% else %} -apikey: {{ elastalert.get('jira_key', '') }} +apikey: {{ elastalert.get('jira_api_key', '') }} {% endif %} \ No newline at end of file diff --git a/salt/elastalert/files/predefined/smtp_auth.yaml b/salt/elastalert/files/predefined/smtp_auth.yaml index 0d488dd5c..d09d101f1 100644 --- a/salt/elastalert/files/predefined/smtp_auth.yaml +++ b/salt/elastalert/files/predefined/smtp_auth.yaml @@ -1,2 +1,2 @@ user: {{ elastalert.get('smtp_user', '') }} -password: {{ elastalert.get('smtp_pass', '') }} \ No newline at end of file +password: {{ elastalert.get('smtp_pass', '') }} diff --git a/salt/elastalert/soc_elastalert.yaml b/salt/elastalert/soc_elastalert.yaml index 81df0541f..ab315c482 100644 --- a/salt/elastalert/soc_elastalert.yaml +++ b/salt/elastalert/soc_elastalert.yaml @@ -43,20 +43,13 @@ elastalert: global: True helpLink: elastalert.html forcedType: string - opsgenie_key: - title: OpsGenie API Key - description: Optional configuration parameter for OpsGenie API Key. Requires a valid Security Onion license key. - global: True - sensitive: True - helpLink: elastalert.html - forcedType: string files: custom: filename__ext: title: Custom Parameter File description: Optional configuration file that can be used to specify custom file contents, such as a SMTP certificate file. When used, the corresponding parameter must be set to this setting's filename.ext path inside the custom subdirectory. For example, if specifying the SMTP cert file, the smtp_cert_file key must be set to /opt/elastalert/custom/smtp.crt in the Alerter Parameters setting for this certificate to be enabled, and assumes this duplicated setting has been named smtp__crt. Note that double underscores will be replaced with a period in the filename. global: True - duplicating: True + duplicates: True file: True helpLink: elastalert.html config: From 03826dd32c02acfdea0a4a94722a5fa853166571 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Tue, 21 May 2024 06:43:07 -0400 Subject: [PATCH 069/101] Update README.md with new Detections screenshot number --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a990326a8..530a21813 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ Hunt ![Hunt](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/56_hunt.png) Detections -![Detections](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/59_detections.png) +![Detections](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/57_detections.png) PCAP ![PCAP](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/62_pcap.png) From d315b95d77d5e9fed89f03d0bb21ada98d74d8a7 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 21 May 2024 07:15:19 -0400 Subject: [PATCH 070/101] elastalert settings --- salt/elastalert/config.sls | 3 -- salt/elastalert/soc_elastalert.yaml | 51 ++++++++++++++++++++++++++--- 2 files changed, 47 insertions(+), 7 deletions(-) diff --git a/salt/elastalert/config.sls b/salt/elastalert/config.sls index c435a212b..25d5bf5f8 100644 --- a/salt/elastalert/config.sls +++ b/salt/elastalert/config.sls @@ -96,10 +96,7 @@ elastacustomsync: - user: 933 - group: 933 - makedirs: True - - template: jinja - file_mode: 660 - - context: - elastalert: {{ ELASTALERTMERGED }} - show_changes: False elastapredefinedsync: diff --git a/salt/elastalert/soc_elastalert.yaml b/salt/elastalert/soc_elastalert.yaml index ab315c482..435c5be6a 100644 --- a/salt/elastalert/soc_elastalert.yaml +++ b/salt/elastalert/soc_elastalert.yaml @@ -45,11 +45,54 @@ elastalert: forcedType: string files: custom: - filename__ext: - title: Custom Parameter File - description: Optional configuration file that can be used to specify custom file contents, such as a SMTP certificate file. When used, the corresponding parameter must be set to this setting's filename.ext path inside the custom subdirectory. For example, if specifying the SMTP cert file, the smtp_cert_file key must be set to /opt/elastalert/custom/smtp.crt in the Alerter Parameters setting for this certificate to be enabled, and assumes this duplicated setting has been named smtp__crt. Note that double underscores will be replaced with a period in the filename. + alertmanager_ca__crt: + description: Optional custom Certificate Authority for connecting to an AlertManager server. To utilize this custom file, the alertmanager_ca_certs key must be set to /opt/elastalert/custom/alertmanager_ca.crt in the Alerter Parameters setting. Requires a valid Security Onion license key. + global: True + file: True + helpLink: elastalert.html + gelf_ca__crt: + description: Optional custom Certificate Authority for connecting to a Graylog server. To utilize this custom file, the graylog_ca_certs key must be set to /opt/elastalert/custom/graylog_ca.crt in the Alerter Parameters setting. Requires a valid Security Onion license key. + global: True + file: True + helpLink: elastalert.html + http_post_ca__crt: + description: Optional custom Certificate Authority for connecting to a generic HTTP server, via the legacy HTTP POST alerter. To utilize this custom file, the http_post_ca_certs key must be set to /opt/elastalert/custom/http_post2_ca.crt in the Alerter Parameters setting. Requires a valid Security Onion license key. + global: True + file: True + helpLink: elastalert.html + http_post2_ca__crt: + description: Optional custom Certificate Authority for connecting to a generic HTTP server, via the newer HTTP POST 2 alerter. To utilize this custom file, the http_post2_ca_certs key must be set to /opt/elastalert/custom/http_post2_ca.crt in the Alerter Parameters setting. Requires a valid Security Onion license key. + global: True + file: True + helpLink: elastalert.html + ms_teams_ca__crt: + description: Optional custom Certificate Authority for connecting to Microsoft Teams server. To utilize this custom file, the ms_teams_ca_certs key must be set to /opt/elastalert/custom/ms_teams_ca.crt in the Alerter Parameters setting. Requires a valid Security Onion license key. + global: True + file: True + helpLink: elastalert.html + pagerduty_ca__crt: + description: Optional custom Certificate Authority for connecting to PagerDuty server. To utilize this custom file, the pagerduty_ca_certs key must be set to /opt/elastalert/custom/pagerduty_ca.crt in the Alerter Parameters setting. Requires a valid Security Onion license key. + global: True + file: True + helpLink: elastalert.html + rocket_chat_ca__crt: + description: Optional custom Certificate Authority for connecting to PagerDuty server. To utilize this custom file, the rocket_chart_ca_certs key must be set to /opt/elastalert/custom/rocket_chat_ca.crt in the Alerter Parameters setting. Requires a valid Security Onion license key. + global: True + file: True + helpLink: elastalert.html + smtp__crt: + description: Optional custom certificate for connecting to an SMTP server. To utilize this custom file, the smtp_cert_file key must be set to /opt/elastalert/custom/smtp.crt in the Alerter Parameters setting. Requires a valid Security Onion license key. + global: True + file: True + helpLink: elastalert.html + smtp__key: + description: Optional custom certificate key for connecting to an SMTP server. To utilize this custom file, the smtp_key_file key must be set to /opt/elastalert/custom/smtp.key in the Alerter Parameters setting. Requires a valid Security Onion license key. + global: True + file: True + helpLink: elastalert.html + slack_ca__crt: + description: Optional custom Certificate Authority for connecting to Slack. To utilize this custom file, the slack_ca_certs key must be set to /opt/elastalert/custom/slack_ca.crt in the Alerter Parameters setting. Requires a valid Security Onion license key. global: True - duplicates: True file: True helpLink: elastalert.html config: From 8ce19a93b96ff04d35f99b0c403bf6dc1ec0651b Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 21 May 2024 13:29:20 -0400 Subject: [PATCH 071/101] exclude false positives related to detections --- salt/common/tools/sbin/so-log-check | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check index 5bee4d254..cf1691589 100755 --- a/salt/common/tools/sbin/so-log-check +++ b/salt/common/tools/sbin/so-log-check @@ -203,6 +203,8 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|context deadline exceeded" EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Error running query:" # Specific issues with detection rules EXCLUDED_ERRORS="$EXCLUDED_ERRORS|detect-parse" # Suricata encountering a malformed rule + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|integrity check failed" # Detections: Exclude false positive due to automated testing + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|syncErrors" # Detections: Not an actual error fi RESULT=0 From d57cc9627f1273401cf25fe515cf9c474100d088 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 21 May 2024 13:31:50 -0400 Subject: [PATCH 072/101] exclude false positives related to detections --- salt/elastalert/defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elastalert/defaults.yaml b/salt/elastalert/defaults.yaml index 8021533ab..393932992 100644 --- a/salt/elastalert/defaults.yaml +++ b/salt/elastalert/defaults.yaml @@ -40,4 +40,4 @@ elastalert: level: INFO handlers: - file - propagate: False \ No newline at end of file + propagate: false From deb140e38e2223aa1d727a78cbf2e73d8a8f1701 Mon Sep 17 00:00:00 2001 From: weslambert Date: Tue, 21 May 2024 13:38:52 -0400 Subject: [PATCH 073/101] Exclude detections from template name matching --- .../tools/sbin_jinja/so-elasticsearch-templates-load | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load index 8f45d6c36..080348522 100755 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load @@ -133,7 +133,7 @@ if [ ! -f $STATE_FILE_SUCCESS ]; then for i in $pattern; do TEMPLATE=${i::-14} COMPONENT_PATTERN=${TEMPLATE:3} - MATCH=$(echo "$TEMPLATE" | grep -E "^so-logs-|^so-metrics" | grep -v osquery) + MATCH=$(echo "$TEMPLATE" | grep -E "^so-logs-|^so-metrics" | grep -vE "detections|osquery") if [[ -n "$MATCH" && ! "$COMPONENT_LIST" =~ "$COMPONENT_PATTERN" ]]; then load_failures=$((load_failures+1)) echo "Component template does not exist for $COMPONENT_PATTERN. The index template will not be loaded. Load failures: $load_failures" From f4490fab58954a864942425c4f909f55d7679ffc Mon Sep 17 00:00:00 2001 From: weslambert Date: Tue, 21 May 2024 17:05:39 -0400 Subject: [PATCH 074/101] Add rule.uuid for YARA matches --- salt/elasticsearch/files/ingest/strelka.file | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/elasticsearch/files/ingest/strelka.file b/salt/elasticsearch/files/ingest/strelka.file index d55e19350..f66a98857 100644 --- a/salt/elasticsearch/files/ingest/strelka.file +++ b/salt/elasticsearch/files/ingest/strelka.file @@ -56,6 +56,7 @@ { "set": { "if": "ctx.exiftool?.Subsystem != null", "field": "host.subsystem", "value": "{{exiftool.Subsystem}}", "ignore_failure": true }}, { "set": { "if": "ctx.scan?.yara?.matches instanceof List", "field": "rule.name", "value": "{{scan.yara.matches.0}}" }}, { "set": { "if": "ctx.rule?.name != null", "field": "event.dataset", "value": "alert", "override": true }}, + { "set": { "if": "ctx.rule?.name != null", "field": "rule.uuid", "value": "rule.name", "override": true }}, { "rename": { "field": "file.flavors.mime", "target_field": "file.mime_type", "ignore_missing": true }}, { "set": { "if": "ctx.rule?.name != null && ctx.rule?.score == null", "field": "event.severity", "value": 3, "override": true } }, { "convert" : { "if": "ctx.rule?.score != null", "field" : "rule.score","type": "integer"}}, From 3992ef108207b88644198cb88e42860f515bf02d Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Tue, 21 May 2024 17:45:56 -0400 Subject: [PATCH 075/101] Add rule.uuid to default groupbys --- salt/soc/defaults.yaml | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 0113f22cc..d19f7ed40 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -2020,6 +2020,7 @@ soc: - event_data.destination.port - event_data.process.executable - event_data.process.pid + - rule.uuid ':sigma:': - soc_timestamp - event.dataset @@ -2033,6 +2034,7 @@ soc: - event_data.destination.port - event_data.process.executable - event_data.process.pid + - rule.uuid ':strelka:': - soc_timestamp - event.dataset @@ -2042,6 +2044,7 @@ soc: - file.source - file.mime_type - log.id.fuid + - rule.uuid queryBaseFilter: tags:alert queryToggleFilters: - name: acknowledged @@ -2056,17 +2059,17 @@ soc: - acknowledged queries: - name: 'Group By Name, Module' - query: '* | groupby rule.name event.module* event.severity_label' + query: '* | groupby rule.name event.module* event.severity_label rule.uuid' - name: 'Group By Sensor, Source IP/Port, Destination IP/Port, Name' - query: '* | groupby observer.name source.ip source.port destination.ip destination.port rule.name network.community_id event.severity_label' + query: '* | groupby observer.name source.ip source.port destination.ip destination.port rule.name network.community_id event.severity_label rule.uuid' - name: 'Group By Source IP, Name' - query: '* | groupby source.ip rule.name event.severity_label' + query: '* | groupby source.ip rule.name event.severity_label rule.uuid' - name: 'Group By Source Port, Name' - query: '* | groupby source.port rule.name event.severity_label' + query: '* | groupby source.port rule.name event.severity_label rule.uuid' - name: 'Group By Destination IP, Name' - query: '* | groupby destination.ip rule.name event.severity_label' + query: '* | groupby destination.ip rule.name event.severity_label rule.uuid' - name: 'Group By Destination Port, Name' - query: '* | groupby destination.port rule.name event.severity_label' + query: '* | groupby destination.port rule.name event.severity_label rule.uuid' - name: Ungroup query: '*' grid: From f9e9b825cffe381115f9aa965f07c084001d866b Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Tue, 21 May 2024 17:53:20 -0400 Subject: [PATCH 076/101] Removed unneeded groupby --- salt/soc/defaults.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index d19f7ed40..15f1fd8ce 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -2020,7 +2020,6 @@ soc: - event_data.destination.port - event_data.process.executable - event_data.process.pid - - rule.uuid ':sigma:': - soc_timestamp - event.dataset @@ -2034,7 +2033,6 @@ soc: - event_data.destination.port - event_data.process.executable - event_data.process.pid - - rule.uuid ':strelka:': - soc_timestamp - event.dataset @@ -2044,7 +2042,6 @@ soc: - file.source - file.mime_type - log.id.fuid - - rule.uuid queryBaseFilter: tags:alert queryToggleFilters: - name: acknowledged From 8af3158ea7bd3cdc7dfe22f1c6103db06ebbd7ac Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 21 May 2024 18:28:21 -0400 Subject: [PATCH 077/101] fix elastalert settings --- salt/elastalert/map.jinja | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/salt/elastalert/map.jinja b/salt/elastalert/map.jinja index 8d4e65652..b8ceca277 100644 --- a/salt/elastalert/map.jinja +++ b/salt/elastalert/map.jinja @@ -15,20 +15,16 @@ {% set ELASTALERTMERGED = salt['pillar.get']('elastalert', ELASTALERTDEFAULTS.elastalert, merge=True) %} {% if 'ntf' in salt['pillar.get']('features', []) %} - {% set params = ELASTALERTMERGED.alerter_parameters | load_yaml %} - {% if params != None %} + {% set params = ELASTALERTMERGED.get('alerter_parameters', '') | load_yaml %} + {% if params != None and params | length > 0 %} {% do ELASTALERTMERGED.config.update(params) %} {% endif %} - {% if ELASTALERTMERGED.smtp_user | length > 0 %} + {% if ELASTALERTMERGED.get('smtp_user', '') | length > 0 %} {% do ELASTALERTMERGED.config.update({'smtp_auth_file': '/opt/elastalert/predefined/smtp_auth.yaml'}) %} {% endif %} - {% if ELASTALERTMERGED.smtp_user | length > 0 %} - {% do ELASTALERTMERGED.config.update({'smtp_auth_file': '/opt/elastalert/predefined/smtp_auth.yaml'}) %} - {% endif %} - - {% if ELASTALERTMERGED.jira_user | length > 0 or ELASTALERTMERGED.jira_key | length > 0 %} + {% if ELASTALERTMERGED.get('jira_user', '') | length > 0 or ELASTALERTMERGED.get('jira_key', '') | length > 0 %} {% do ELASTALERTMERGED.config.update({'jira_account_file': '/opt/elastalert/predefined/jira_auth.yaml'}) %} {% endif %} From 3cfd71075602fa04b104f004a22cbf151258f970 Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 22 May 2024 13:41:32 -0400 Subject: [PATCH 078/101] Change tab casing to be consistent with other whiptail prompts --- setup/so-whiptail | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index 06d62a027..1dab63237 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -475,7 +475,7 @@ whiptail_end_settings() { read -r -d '' msg <<-EOM $end_msg - Press the Tab key to select yes or no. + Press the TAB key to select yes or no. EOM whiptail --title "The following options have been set, would you like to proceed?" --yesno "$msg" 24 75 --scrolltext From d19c1a514bee994b03a3f3f54df531bbcabcbd7c Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Wed, 22 May 2024 15:12:23 -0400 Subject: [PATCH 079/101] Detections backup script --- salt/soc/config.sls | 20 ++++ salt/soc/files/soc/so-detections-backup.py | 111 +++++++++++++++++++++ 2 files changed, 131 insertions(+) create mode 100644 salt/soc/files/soc/so-detections-backup.py diff --git a/salt/soc/config.sls b/salt/soc/config.sls index a85032295..65d6bd2fa 100644 --- a/salt/soc/config.sls +++ b/salt/soc/config.sls @@ -80,6 +80,15 @@ socmotd: - mode: 600 - template: jinja +filedetectionsbackup: + file.managed: + - name: /opt/so/conf/soc/so-detections-backup.py + - source: salt://soc/files/soc/so-detections-backup.py + - user: 939 + - group: 939 + - mode: 600 + - show_changes: False + crondetectionsruntime: cron.present: - name: /usr/sbin/so-detections-runtime-status cron @@ -91,6 +100,17 @@ crondetectionsruntime: - month: '*' - dayweek: '*' +crondetectionsbackup: + cron.present: + - name: python3 /opt/so/conf/soc/so-detections-backup.py + - identifier: detections-backup + - user: root + - minute: '0' + - hour: '0' + - daymonth: '*' + - month: '*' + - dayweek: '*' + socsigmafinalpipeline: file.managed: - name: /opt/so/conf/soc/sigma_final_pipeline.yaml diff --git a/salt/soc/files/soc/so-detections-backup.py b/salt/soc/files/soc/so-detections-backup.py new file mode 100644 index 000000000..b7e6e2491 --- /dev/null +++ b/salt/soc/files/soc/so-detections-backup.py @@ -0,0 +1,111 @@ +# Copyright 2020-2023 Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +# This script queries Elasticsearch for Custom Detections and all Overrides, +# and git commits them to disk at $OUTPUT_DIR + +import os +import subprocess +import json +import requests +from requests.auth import HTTPBasicAuth +import urllib3 +from datetime import datetime + +# Suppress SSL warnings +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + +# Constants +ES_URL = "https://localhost:9200/so-detection/_search" +QUERY_DETECTIONS = '{"query": {"bool": {"must": [{"match_all": {}}, {"term": {"so_detection.ruleset": "__custom__"}}]}},"size": 10000}' +QUERY_OVERRIDES = '{"query": {"bool": {"must": [{"exists": {"field": "so_detection.overrides"}}]}},"size": 10000}' +OUTPUT_DIR = "/nsm/backup/detections/repo" +AUTH_FILE = "/opt/so/conf/elasticsearch/curl.config" + +def get_auth_credentials(auth_file): + with open(auth_file, 'r') as file: + for line in file: + if line.startswith('user ='): + return line.split('=', 1)[1].strip().replace('"', '') + +def query_elasticsearch(query, auth): + headers = {"Content-Type": "application/json"} + response = requests.get(ES_URL, headers=headers, data=query, auth=auth, verify=False) + response.raise_for_status() + return response.json() + +def save_content(hit, base_folder, subfolder="", extension="txt"): + so_detection = hit["_source"]["so_detection"] + public_id = so_detection["publicId"] + content = so_detection["content"] + file_dir = os.path.join(base_folder, subfolder) + os.makedirs(file_dir, exist_ok=True) + file_path = os.path.join(file_dir, f"{public_id}.{extension}") + with open(file_path, "w") as f: + f.write(content) + return file_path + +def save_overrides(hit): + so_detection = hit["_source"]["so_detection"] + public_id = so_detection["publicId"] + overrides = so_detection["overrides"] + language = so_detection["language"] + folder = os.path.join(OUTPUT_DIR, language, "overrides") + os.makedirs(folder, exist_ok=True) + extension = "yaml" if language == "sigma" else "txt" + file_path = os.path.join(folder, f"{public_id}.{extension}") + with open(file_path, "w") as f: + f.write('\n'.join(json.dumps(override) for override in overrides) if isinstance(overrides, list) else overrides) + return file_path + +def ensure_git_repo(): + if not os.path.isdir(os.path.join(OUTPUT_DIR, '.git')): + subprocess.run(["git", "config", "--global", "init.defaultBranch", "main"], check=True) + subprocess.run(["git", "-C", OUTPUT_DIR, "init"], check=True) + subprocess.run(["git", "-C", OUTPUT_DIR, "remote", "add", "origin", "default"], check=True) + +def commit_changes(): + ensure_git_repo() + subprocess.run(["git", "-C", OUTPUT_DIR, "config", "user.email", "securityonion@local.invalid"], check=True) + subprocess.run(["git", "-C", OUTPUT_DIR, "config", "user.name", "securityonion"], check=True) + subprocess.run(["git", "-C", OUTPUT_DIR, "add", "."], check=True) + status_result = subprocess.run(["git", "-C", OUTPUT_DIR, "status"], capture_output=True, text=True) + print(status_result.stdout) + commit_result = subprocess.run(["git", "-C", OUTPUT_DIR, "commit", "-m", "Update detections and overrides"], check=False, capture_output=True) + if commit_result.returncode == 1: + print("No changes to commit.") + elif commit_result.returncode == 0: + print("Changes committed successfully.") + else: + commit_result.check_returncode() + +def main(): + try: + timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + print(f"Backing up Custom Detections and all Overrides to {OUTPUT_DIR} - {timestamp}\n") + + auth_credentials = get_auth_credentials(AUTH_FILE) + username, password = auth_credentials.split(':', 1) + auth = HTTPBasicAuth(username, password) + + # Query and save custom detections + detections = query_elasticsearch(QUERY_DETECTIONS, auth)["hits"]["hits"] + for hit in detections: + save_content(hit, OUTPUT_DIR, hit["_source"]["so_detection"]["language"], "yaml" if hit["_source"]["so_detection"]["language"] == "sigma" else "txt") + + # Query and save overrides + overrides = query_elasticsearch(QUERY_OVERRIDES, auth)["hits"]["hits"] + for hit in overrides: + save_overrides(hit) + + commit_changes() + + timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + print(f"Backup Completed - {timestamp}") + except Exception as e: + print(f"An error occurred: {e}") + +if __name__ == "__main__": + main() \ No newline at end of file From a072e34cfeaaabdb24090f6c99eb2bf4b8057b88 Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Wed, 22 May 2024 17:12:41 -0400 Subject: [PATCH 080/101] Fix casing issue --- salt/soc/soc_soc.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index 78f541982..60de637b4 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -200,7 +200,7 @@ soc: global: True advanced: True helpLink: yara.html - autoEnabledYARARules: + autoEnabledYaraRules: description: 'YARA rules to automatically enable on initial import. Format is $Ruleset - for example, for the default shipped ruleset: securityonion-yara' global: True advanced: True From 3d4f3a04a3c855e758f00ecbfa0fd9376607f8aa Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Thu, 23 May 2024 05:56:18 -0400 Subject: [PATCH 081/101] Update defaults.yaml to fix order of groupby tables and eliminate duplicate --- salt/soc/defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 15f1fd8ce..6ecf1f183 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1817,7 +1817,7 @@ soc: query: 'tags:dhcp | groupby host.hostname | groupby -sankey host.hostname client.address | groupby client.address | groupby -sankey client.address server.address | groupby server.address | groupby dhcp.message_types | groupby host.domain' - name: DNS description: DNS (Domain Name System) queries - query: 'tags:dns | groupby dns.query.name | groupby source.ip | groupby -sankey source.ip destination.ip | groupby destination.ip | groupby destination.port | groupby dns.highest_registered_domain | groupby dns.parent_domain | groupby dns.response.code_name | groupby dns.answers.name | groupby dns.query.type_name | groupby dns.response.code_name | groupby destination_geo.organization_name' + query: 'tags:dns | groupby dns.query.name | groupby source.ip | groupby -sankey source.ip destination.ip | groupby destination.ip | groupby destination.port | groupby dns.highest_registered_domain | groupby dns.parent_domain | groupby dns.query.type_name | groupby dns.response.code_name | groupby dns.answers.name | groupby destination_geo.organization_name' - name: DPD description: DPD (Dynamic Protocol Detection) errors query: 'tags:dpd | groupby error.reason | groupby -sankey error.reason source.ip | groupby source.ip | groupby -sankey source.ip destination.ip | groupby destination.ip | groupby destination.port | groupby network.protocol | groupby destination_geo.organization_name' From 8e7c487cb0e0e13a28ea71896dd2b52fb2ab1ac6 Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Thu, 23 May 2024 05:59:31 -0400 Subject: [PATCH 082/101] Fix strelka rule.uuid --- salt/elasticsearch/files/ingest/strelka.file | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/files/ingest/strelka.file b/salt/elasticsearch/files/ingest/strelka.file index f66a98857..d9aed3b29 100644 --- a/salt/elasticsearch/files/ingest/strelka.file +++ b/salt/elasticsearch/files/ingest/strelka.file @@ -56,7 +56,7 @@ { "set": { "if": "ctx.exiftool?.Subsystem != null", "field": "host.subsystem", "value": "{{exiftool.Subsystem}}", "ignore_failure": true }}, { "set": { "if": "ctx.scan?.yara?.matches instanceof List", "field": "rule.name", "value": "{{scan.yara.matches.0}}" }}, { "set": { "if": "ctx.rule?.name != null", "field": "event.dataset", "value": "alert", "override": true }}, - { "set": { "if": "ctx.rule?.name != null", "field": "rule.uuid", "value": "rule.name", "override": true }}, + { "set": { "if": "ctx.rule?.name != null", "field": "rule.uuid", "value": "{{rule.name}}", "override": true }}, { "rename": { "field": "file.flavors.mime", "target_field": "file.mime_type", "ignore_missing": true }}, { "set": { "if": "ctx.rule?.name != null && ctx.rule?.score == null", "field": "event.severity", "value": 3, "override": true } }, { "convert" : { "if": "ctx.rule?.score != null", "field" : "rule.score","type": "integer"}}, From 1e6161f89c82277a409d0051f82a0fa5a79582d9 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 23 May 2024 08:19:43 -0400 Subject: [PATCH 083/101] Update defaults.yaml --- salt/soc/defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 6ecf1f183..86170b4ce 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1482,7 +1482,7 @@ soc: showSubtitle: true - name: Elastalerts description: '' - query: '_type:elastalert | groupby rule.name' + query: 'event.dataset:sigma.alert | groupby rule.name' showSubtitle: true - name: Alerts description: Show all alerts grouped by alert source From 0b9ebefdb670e9ad85bd4bb415291e82ef71c505 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 23 May 2024 10:08:23 -0400 Subject: [PATCH 084/101] only show telem status in final whiptail if new deployment --- setup/so-whiptail | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index 1dab63237..4fab6dbe4 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -451,7 +451,7 @@ whiptail_end_settings() { done fi - if [[ ! $is_airgap ]]; then + if [[ ! $is_airgap ]] && [[ $dist_option == "NEWDEPLOYMENT" ]]; then if [[ $telemetry -eq 0 ]]; then __append_end_msg "SOC Telemetry: enabled" else From ea7715f7297c426956b7074193ca331f0aaa0b18 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 23 May 2024 10:41:10 -0400 Subject: [PATCH 085/101] use waitforstate var instead. --- setup/so-whiptail | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index 4fab6dbe4..d950f2921 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -451,7 +451,7 @@ whiptail_end_settings() { done fi - if [[ ! $is_airgap ]] && [[ $dist_option == "NEWDEPLOYMENT" ]]; then + if [[ ! $is_airgap ]] && [[ $waitforstate ]]; then if [[ $telemetry -eq 0 ]]; then __append_end_msg "SOC Telemetry: enabled" else From 15155613c3d60a2e48a7e0c922ae09eb5d225acb Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 24 May 2024 08:23:45 -0400 Subject: [PATCH 086/101] provide default columns when viewing SOC logs --- salt/soc/defaults.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 86170b4ce..39960d946 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1271,6 +1271,14 @@ soc: - netflow.type - netflow.exporter.version - observer.ip + ':soc:': + - soc_timestamp + - source.ip + - soc.fields.requestMethod + - soc.fields.requestPath + - soc.fields.statusCode + - event.action + - soc.fields.error server: bindAddress: 0.0.0.0:9822 baseUrl: / From bd11d59c15fb711332ab0c67423e217f2b4b1b70 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 24 May 2024 08:38:12 -0400 Subject: [PATCH 087/101] add event.dataset since there are other datasets in soc logs --- salt/soc/defaults.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 39960d946..9f5faf50b 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1273,6 +1273,7 @@ soc: - observer.ip ':soc:': - soc_timestamp + - event.dataset - source.ip - soc.fields.requestMethod - soc.fields.requestPath From 66725b11b304d1742caad49854bd968f71196c97 Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Fri, 24 May 2024 09:55:10 -0400 Subject: [PATCH 088/101] Added unit tests --- salt/soc/config.sls | 3 +- salt/soc/files/soc/so-detections-backup.py | 2 + .../files/soc/so-detections-backup_test.py | 159 ++++++++++++++++++ 3 files changed, 162 insertions(+), 2 deletions(-) create mode 100644 salt/soc/files/soc/so-detections-backup_test.py diff --git a/salt/soc/config.sls b/salt/soc/config.sls index 65d6bd2fa..8d1f0f694 100644 --- a/salt/soc/config.sls +++ b/salt/soc/config.sls @@ -87,7 +87,6 @@ filedetectionsbackup: - user: 939 - group: 939 - mode: 600 - - show_changes: False crondetectionsruntime: cron.present: @@ -102,7 +101,7 @@ crondetectionsruntime: crondetectionsbackup: cron.present: - - name: python3 /opt/so/conf/soc/so-detections-backup.py + - name: python3 /opt/so/conf/soc/so-detections-backup.py &>> /opt/so/log/soc/detections-backup.log - identifier: detections-backup - user: root - minute: '0' diff --git a/salt/soc/files/soc/so-detections-backup.py b/salt/soc/files/soc/so-detections-backup.py index b7e6e2491..085b1e4c7 100644 --- a/salt/soc/files/soc/so-detections-backup.py +++ b/salt/soc/files/soc/so-detections-backup.py @@ -86,6 +86,8 @@ def main(): timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") print(f"Backing up Custom Detections and all Overrides to {OUTPUT_DIR} - {timestamp}\n") + os.makedirs(OUTPUT_DIR, exist_ok=True) + auth_credentials = get_auth_credentials(AUTH_FILE) username, password = auth_credentials.split(':', 1) auth = HTTPBasicAuth(username, password) diff --git a/salt/soc/files/soc/so-detections-backup_test.py b/salt/soc/files/soc/so-detections-backup_test.py new file mode 100644 index 000000000..3afa11886 --- /dev/null +++ b/salt/soc/files/soc/so-detections-backup_test.py @@ -0,0 +1,159 @@ +# Copyright 2020-2023 Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +import unittest +from unittest.mock import patch, MagicMock, mock_open, call +import requests +import os +import subprocess +import json +from datetime import datetime +import importlib + +ds = importlib.import_module('so-detections-backup') + +class TestBackupScript(unittest.TestCase): + + def setUp(self): + self.output_dir = '/nsm/backup/detections/repo' + self.auth_file_path = '/nsm/backup/detections/repo' + self.mock_auth_data = 'user = "so_elastic:@Tu_dv_[7SvK7[-JZN39BBlSa;WAyf8rCY+3w~Sntp=7oR9*~34?Csi)a@v?)K*vK4vQAywS"' + self.auth_credentials = 'so_elastic:@Tu_dv_[7SvK7[-JZN39BBlSa;WAyf8rCY+3w~Sntp=7oR9*~34?Csi)a@v?)K*vK4vQAywS' + self.auth = requests.auth.HTTPBasicAuth('so_elastic', '@Tu_dv_[7SvK7[-JZN39BBlSa;WAyf8rCY+3w~Sntp=7oR9*~34?Csi)a@v?)K*vK4vQAywS') + self.mock_detection_hit = { + "_source": { + "so_detection": { + "publicId": "test_id", + "content": "test_content", + "language": "suricata" + } + } + } + self.mock_override_hit = { + "_source": { + "so_detection": { + "publicId": "test_id", + "overrides": [{"key": "value"}], + "language": "sigma" + } + } + } + + def assert_file_written(self, mock_file, expected_path, expected_content): + mock_file.assert_called_once_with(expected_path, 'w') + mock_file().write.assert_called_once_with(expected_content) + + @patch('builtins.open', new_callable=mock_open, read_data='user = "so_elastic:@Tu_dv_[7SvK7[-JZN39BBlSa;WAyf8rCY+3w~Sntp=7oR9*~34?Csi)a@v?)K*vK4vQAywS"') + def test_get_auth_credentials(self, mock_file): + credentials = ds.get_auth_credentials(self.auth_file_path) + self.assertEqual(credentials, self.auth_credentials) + mock_file.assert_called_once_with(self.auth_file_path, 'r') + + @patch('requests.get') + def test_query_elasticsearch(self, mock_get): + mock_response = MagicMock() + mock_response.json.return_value = {'hits': {'hits': []}} + mock_response.raise_for_status = MagicMock() + mock_get.return_value = mock_response + + response = ds.query_elasticsearch(ds.QUERY_DETECTIONS, self.auth) + + self.assertEqual(response, {'hits': {'hits': []}}) + mock_get.assert_called_once_with( + ds.ES_URL, + headers={"Content-Type": "application/json"}, + data=ds.QUERY_DETECTIONS, + auth=self.auth, + verify=False + ) + + @patch('os.makedirs') + @patch('builtins.open', new_callable=mock_open) + def test_save_content(self, mock_file, mock_makedirs): + file_path = ds.save_content(self.mock_detection_hit, self.output_dir, 'subfolder', 'txt') + expected_path = f'{self.output_dir}/subfolder/test_id.txt' + self.assertEqual(file_path, expected_path) + mock_makedirs.assert_called_once_with(f'{self.output_dir}/subfolder', exist_ok=True) + self.assert_file_written(mock_file, expected_path, 'test_content') + + @patch('os.makedirs') + @patch('builtins.open', new_callable=mock_open) + def test_save_overrides(self, mock_file, mock_makedirs): + file_path = ds.save_overrides(self.mock_override_hit) + expected_path = f'{self.output_dir}/sigma/overrides/test_id.yaml' + self.assertEqual(file_path, expected_path) + mock_makedirs.assert_called_once_with(f'{self.output_dir}/sigma/overrides', exist_ok=True) + self.assert_file_written(mock_file, expected_path, json.dumps({"key": "value"})) + + @patch('subprocess.run') + def test_ensure_git_repo(self, mock_run): + mock_run.return_value = MagicMock(returncode=0) + + ds.ensure_git_repo() + + mock_run.assert_has_calls([ + call(["git", "config", "--global", "init.defaultBranch", "main"], check=True), + call(["git", "-C", self.output_dir, "init"], check=True), + call(["git", "-C", self.output_dir, "remote", "add", "origin", "default"], check=True) + ]) + + @patch('subprocess.run') + def test_commit_changes(self, mock_run): + mock_status_result = MagicMock() + mock_status_result.stdout = "On branch main\nnothing to commit, working tree clean" + mock_commit_result = MagicMock(returncode=1) + # Ensure sufficient number of MagicMock instances for each subprocess.run call + mock_run.side_effect = [mock_status_result, mock_commit_result, MagicMock(returncode=0), MagicMock(returncode=0), MagicMock(returncode=0), MagicMock(returncode=0), MagicMock(returncode=0), MagicMock(returncode=0)] + + print("Running test_commit_changes...") + ds.commit_changes() + print("Finished test_commit_changes.") + + mock_run.assert_has_calls([ + call(["git", "-C", self.output_dir, "config", "user.email", "securityonion@local.invalid"], check=True), + call(["git", "-C", self.output_dir, "config", "user.name", "securityonion"], check=True), + call(["git", "-C", self.output_dir, "add", "."], check=True), + call(["git", "-C", self.output_dir, "status"], capture_output=True, text=True), + call(["git", "-C", self.output_dir, "commit", "-m", "Update detections and overrides"], check=False, capture_output=True) + ]) + + @patch('builtins.print') + @patch('so-detections-backup.commit_changes') + @patch('so-detections-backup.save_overrides') + @patch('so-detections-backup.save_content') + @patch('so-detections-backup.query_elasticsearch') + @patch('so-detections-backup.get_auth_credentials') + @patch('os.makedirs') + def test_main(self, mock_makedirs, mock_get_auth, mock_query, mock_save_content, mock_save_overrides, mock_commit, mock_print): + mock_get_auth.return_value = self.auth_credentials + mock_query.side_effect = [ + {'hits': {'hits': [{"_source": {"so_detection": {"publicId": "1", "content": "content1", "language": "sigma"}}}]}}, + {'hits': {'hits': [{"_source": {"so_detection": {"publicId": "2", "overrides": [{"key": "value"}], "language": "suricata"}}}]}} + ] + + with patch('datetime.datetime') as mock_datetime: + mock_datetime.now.return_value.strftime.return_value = "2024-05-23 20:49:44" + ds.main() + + mock_makedirs.assert_called_once_with(self.output_dir, exist_ok=True) + mock_get_auth.assert_called_once_with(ds.AUTH_FILE) + mock_query.assert_has_calls([ + call(ds.QUERY_DETECTIONS, self.auth), + call(ds.QUERY_OVERRIDES, self.auth) + ]) + mock_save_content.assert_called_once_with( + {"_source": {"so_detection": {"publicId": "1", "content": "content1", "language": "sigma"}}}, + self.output_dir, + "sigma", + "yaml" + ) + mock_save_overrides.assert_called_once_with( + {"_source": {"so_detection": {"publicId": "2", "overrides": [{"key": "value"}], "language": "suricata"}}} + ) + mock_commit.assert_called_once() + mock_print.assert_called() + +if __name__ == '__main__': + unittest.main(verbosity=2) From 4344988abeafe731e26f0305cdd62af1034c4e59 Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Fri, 24 May 2024 12:54:36 -0400 Subject: [PATCH 089/101] Add instructions for sigma and yara repos --- salt/soc/soc_soc.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index 60de637b4..415829460 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -119,14 +119,14 @@ soc: advanced: True rulesRepos: default: &eerulesRepos - description: "Custom Git repos to pull Sigma rules from. 'license' field is required, 'folder' is optional. 'community' disables some management options for the imported rules - they can't be deleted or edited, just tuned, duplicated and Enabled | Disabled." + description: "Custom Git repositories to pull Sigma rules from. 'license' field is required, 'folder' is optional. 'community' disables some management options for the imported rules - they can't be deleted or edited, just tuned, duplicated and Enabled | Disabled. The new settings will be applied within 15 minutes. At that point, you will need to wait for the scheduled rule update to take place (by default, every 24 hours), or you can force the update by nagivating to Detections --> Options dropdown menu --> Elastalert --> Full Update." global: True advanced: True forcedType: "[]{}" helpLink: sigma.html airgap: *eerulesRepos sigmaRulePackages: - description: 'Defines the Sigma Community Ruleset you want to run. One of these (core | core+ | core++ | all ) as well as an optional Add-on (emerging_threats_addon). Once you have changed the ruleset here, you will need to wait for the scheduled rule update to take place (by default, every 24 hours), or you can force the update by nagivating to Detections --> Options dropdown menu --> Elastalert --> Full Update. WARNING! Changing the ruleset will remove all existing non-overlapping Sigma rules of the previous ruleset and their associated overrides. This removal cannot be undone.' + description: 'Defines the Sigma Community Ruleset you want to run. One of these (core | core+ | core++ | all ) as well as an optional Add-on (emerging_threats_addon). Once you have changed the ruleset here, the new settings will be applied within 15 minutes. At that point, you will need to wait for the scheduled rule update to take place (by default, every 24 hours), or you can force the update by nagivating to Detections --> Options dropdown menu --> Elastalert --> Full Update. WARNING! Changing the ruleset will remove all existing non-overlapping Sigma rules of the previous ruleset and their associated overrides. This removal cannot be undone.' global: True advanced: False helpLink: sigma.html @@ -221,7 +221,7 @@ soc: advanced: True rulesRepos: default: &serulesRepos - description: "Custom Git repos to pull YARA rules from. 'license' field is required, 'folder' is optional. 'community' disables some management options for the imported rules - they can't be deleted or edited, just tuned, duplicated and Enabled | Disabled." + description: "Custom Git repositories to pull YARA rules from. 'license' field is required, 'folder' is optional. 'community' disables some management options for the imported rules - they can't be deleted or edited, just tuned, duplicated and Enabled | Disabled. The new settings will be applied within 15 minutes. At that point, you will need to wait for the scheduled rule update to take place (by default, every 24 hours), or you can force the update by nagivating to Detections --> Options dropdown menu --> Strelka --> Full Update."" global: True advanced: True forcedType: "[]{}" From f90d40b4714073c640bccfa05e36755f6a31c6af Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Fri, 24 May 2024 12:56:17 -0400 Subject: [PATCH 090/101] Fix typo --- salt/soc/soc_soc.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index 415829460..1f64eb0bc 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -221,7 +221,7 @@ soc: advanced: True rulesRepos: default: &serulesRepos - description: "Custom Git repositories to pull YARA rules from. 'license' field is required, 'folder' is optional. 'community' disables some management options for the imported rules - they can't be deleted or edited, just tuned, duplicated and Enabled | Disabled. The new settings will be applied within 15 minutes. At that point, you will need to wait for the scheduled rule update to take place (by default, every 24 hours), or you can force the update by nagivating to Detections --> Options dropdown menu --> Strelka --> Full Update."" + description: "Custom Git repositories to pull YARA rules from. 'license' field is required, 'folder' is optional. 'community' disables some management options for the imported rules - they can't be deleted or edited, just tuned, duplicated and Enabled | Disabled. The new settings will be applied within 15 minutes. At that point, you will need to wait for the scheduled rule update to take place (by default, every 24 hours), or you can force the update by nagivating to Detections --> Options dropdown menu --> Strelka --> Full Update." global: True advanced: True forcedType: "[]{}" From 550b3ee92d02d4d6ce6bb11610532b68fb02089c Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Fri, 24 May 2024 14:46:24 -0400 Subject: [PATCH 091/101] Add IDH mappings --- salt/soc/files/soc/sigma_so_pipeline.yaml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/salt/soc/files/soc/sigma_so_pipeline.yaml b/salt/soc/files/soc/sigma_so_pipeline.yaml index 312d07965..8314361f5 100644 --- a/salt/soc/files/soc/sigma_so_pipeline.yaml +++ b/salt/soc/files/soc/sigma_so_pipeline.yaml @@ -17,6 +17,16 @@ transformations: dst_ip: destination.ip.keyword dst_port: destination.port winlog.event_data.User: user.name + logtype: event.code # OpenCanary + # Maps "opencanary" product to SO IDH logs + - id: opencanary_idh_add-fields + type: add_condition + conditions: + event.module: 'opencanary' + event.dataset: 'opencanary.idh' + rule_conditions: + - type: logsource + product: opencanary # Maps "antivirus" category to Windows Defender logs shipped by Elastic Agent Winlog Integration # winlog.event_data.threat_name has to be renamed prior to ingestion, it is originally winlog.event_data.Threat Name - id: antivirus_field-mappings_windows-defender @@ -88,3 +98,11 @@ transformations: - type: logsource product: linux service: auth + # event.code should always be a string + - id: convert_event_code_to_string + type: convert_type + target_type: 'str' + field_name_conditions: + - type: include_fields + fields: + - event.code From 58b565558df8acf341d2419fb27fb92e8725056e Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Fri, 24 May 2024 16:21:59 -0400 Subject: [PATCH 092/101] Dont bail - just wait for enter --- salt/manager/tools/sbin/soup | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 525fce3f6..b6bf61d2a 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -680,16 +680,15 @@ playbook_migration() { active_rules_count=$(find /opt/so/rules/elastalert/playbook/ -type f -name "*.yaml" | wc -l) if [[ "$active_rules_count" -gt 0 ]]; then - # Prompt the user to AGREE if active Elastalert rules found + # Prompt the user to press ENTER if active Elastalert rules found echo echo "$active_rules_count Active Elastalert/Playbook rules found." echo "In preparation for the new Detections module, they will be backed up and then disabled." echo - echo "If you would like to proceed, then type AGREE and press ENTER." + echo "Press ENTER to proceed." echo # Read user input - read INPUT - if [ "${INPUT^^}" != 'AGREE' ]; then fail "SOUP canceled."; fi + read -r echo "Backing up the Elastalert rules..." rsync -av --stats /opt/so/rules/elastalert/playbook/*.yaml /nsm/backup/detections-migration/elastalert/ From 81ee60e6586f2fac5e47c96c40fa2ca32245f6c6 Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Tue, 28 May 2024 06:42:18 -0400 Subject: [PATCH 093/101] Backup .yml files too --- salt/manager/tools/sbin/soup | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index b6bf61d2a..c37138e19 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -677,7 +677,7 @@ playbook_migration() { if grep -A 1 'playbook:' /opt/so/saltstack/local/pillar/minions/* | grep -q 'enabled: True'; then # Check for active Elastalert rules - active_rules_count=$(find /opt/so/rules/elastalert/playbook/ -type f -name "*.yaml" | wc -l) + active_rules_count=$(find /opt/so/rules/elastalert/playbook/ -type f \( -name "*.yaml" -o -name "*.yml" \) | wc -l) if [[ "$active_rules_count" -gt 0 ]]; then # Prompt the user to press ENTER if active Elastalert rules found @@ -691,7 +691,8 @@ playbook_migration() { read -r echo "Backing up the Elastalert rules..." - rsync -av --stats /opt/so/rules/elastalert/playbook/*.yaml /nsm/backup/detections-migration/elastalert/ + rsync -av --stats /opt/so/rules/elastalert/playbook/*.{yaml,yml} /nsm/backup/detections-migration/elastalert/ + fi # Verify that rsync completed successfully if [[ $? -eq 0 ]]; then From 74dfc25376d845d042cfd1c00b57541ec40cffcc Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Tue, 28 May 2024 09:29:10 -0400 Subject: [PATCH 094/101] backup local rules --- salt/manager/tools/sbin/soup | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index c37138e19..9b61f3c8c 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -660,6 +660,11 @@ suricata_idstools_migration() { fail "Error: rsync failed to copy the files. Thresholds have not been backed up." fi + #Backup local rules + mkdir -p /nsm/backup/detections-migration/suricata/local-rules + rsync -av /opt/so/rules/nids/suri/local.rules /nsm/backup/detections-migration/suricata/local-rules + rsync -av /opt/so/saltstack/local/salt/idstools/rules/local.rules /nsm/backup/detections-migration/suricata/local-rules + #Tell SOC to migrate mkdir -p /opt/so/conf/soc/migrations echo "0" > /opt/so/conf/soc/migrations/suricata-migration-2.4.70 From 2a2b86ebe62b2690b4c2b0e51d0124f60c976b24 Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Tue, 28 May 2024 09:43:45 -0400 Subject: [PATCH 095/101] Dont overwrite --- salt/manager/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 9b61f3c8c..7d94fcf55 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -663,7 +663,7 @@ suricata_idstools_migration() { #Backup local rules mkdir -p /nsm/backup/detections-migration/suricata/local-rules rsync -av /opt/so/rules/nids/suri/local.rules /nsm/backup/detections-migration/suricata/local-rules - rsync -av /opt/so/saltstack/local/salt/idstools/rules/local.rules /nsm/backup/detections-migration/suricata/local-rules + rsync -av /opt/so/saltstack/local/salt/idstools/rules/local.rules /nsm/backup/detections-migration/suricata/local-rules/local.rules.bak #Tell SOC to migrate mkdir -p /opt/so/conf/soc/migrations From f68ac23f0eb69334e78fb0c8c45ddd71f6a7f8d4 Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Tue, 28 May 2024 10:03:31 -0400 Subject: [PATCH 096/101] Fix fi Signed-off-by: DefensiveDepth --- salt/manager/tools/sbin/soup | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 7d94fcf55..39c684c08 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -697,7 +697,6 @@ playbook_migration() { echo "Backing up the Elastalert rules..." rsync -av --stats /opt/so/rules/elastalert/playbook/*.{yaml,yml} /nsm/backup/detections-migration/elastalert/ - fi # Verify that rsync completed successfully if [[ $? -eq 0 ]]; then From ee4ca0d7a214ee764f841d0d90e9119ced80fe07 Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Tue, 28 May 2024 10:24:09 -0400 Subject: [PATCH 097/101] Check to see if local exists --- salt/manager/tools/sbin/soup | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 39c684c08..b8297ad44 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -663,7 +663,9 @@ suricata_idstools_migration() { #Backup local rules mkdir -p /nsm/backup/detections-migration/suricata/local-rules rsync -av /opt/so/rules/nids/suri/local.rules /nsm/backup/detections-migration/suricata/local-rules - rsync -av /opt/so/saltstack/local/salt/idstools/rules/local.rules /nsm/backup/detections-migration/suricata/local-rules/local.rules.bak + if [[ -f /opt/so/saltstack/local/salt/idstools/rules/local.rules ]]; then + rsync -av /opt/so/saltstack/local/salt/idstools/rules/local.rules /nsm/backup/detections-migration/suricata/local-rules/local.rules.bak + fi #Tell SOC to migrate mkdir -p /opt/so/conf/soc/migrations From 0d034e7adcacc414cce6531f95e94da65ee9f56e Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Wed, 29 May 2024 10:55:56 -0400 Subject: [PATCH 098/101] fix rsync --- salt/manager/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index b8297ad44..1e9585987 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -698,7 +698,7 @@ playbook_migration() { read -r echo "Backing up the Elastalert rules..." - rsync -av --stats /opt/so/rules/elastalert/playbook/*.{yaml,yml} /nsm/backup/detections-migration/elastalert/ + rsync -av --ignore-missing-args --stats /opt/so/rules/elastalert/playbook/*.{yaml,yml} /nsm/backup/detections-migration/elastalert/ # Verify that rsync completed successfully if [[ $? -eq 0 ]]; then From e98b8566c942a4557b9ab33ced61524f4d2e725c Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 29 May 2024 14:50:22 -0400 Subject: [PATCH 099/101] 2.4.70 --- DOWNLOAD_AND_VERIFY_ISO.md | 22 ++++++++++----------- sigs/securityonion-2.4.70-20240529.iso.sig | Bin 0 -> 566 bytes 2 files changed, 11 insertions(+), 11 deletions(-) create mode 100644 sigs/securityonion-2.4.70-20240529.iso.sig diff --git a/DOWNLOAD_AND_VERIFY_ISO.md b/DOWNLOAD_AND_VERIFY_ISO.md index 4493f210d..fcefce469 100644 --- a/DOWNLOAD_AND_VERIFY_ISO.md +++ b/DOWNLOAD_AND_VERIFY_ISO.md @@ -1,17 +1,17 @@ -### 2.4.60-20240320 ISO image released on 2024/03/20 +### 2.4.70-20240529 ISO image released on 2024/03/20 ### Download and Verify -2.4.60-20240320 ISO image: -https://download.securityonion.net/file/securityonion/securityonion-2.4.60-20240320.iso +2.4.70-20240529 ISO image: +https://download.securityonion.net/file/securityonion/securityonion-2.4.70-20240529.iso -MD5: 178DD42D06B2F32F3870E0C27219821E -SHA1: 73EDCD50817A7F6003FE405CF1808A30D034F89D -SHA256: DD334B8D7088A7B78160C253B680D645E25984BA5CCAB5CC5C327CA72137FC06 +MD5: 8FCCF31C2470D1ABA380AF196B611DEC +SHA1: EE5E8F8C14819E7A1FE423E6920531A97F39600B +SHA256: EF5E781D50D50660F452ADC54FD4911296ECBECED7879FA8E04687337CA89BEC Signature for ISO image: -https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.60-20240320.iso.sig +https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.70-20240529.iso.sig Signing key: https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.4/main/KEYS @@ -25,22 +25,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2. Download the signature file for the ISO: ``` -wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.60-20240320.iso.sig +wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.70-20240529.iso.sig ``` Download the ISO image: ``` -wget https://download.securityonion.net/file/securityonion/securityonion-2.4.60-20240320.iso +wget https://download.securityonion.net/file/securityonion/securityonion-2.4.70-20240529.iso ``` Verify the downloaded ISO image using the signature file: ``` -gpg --verify securityonion-2.4.60-20240320.iso.sig securityonion-2.4.60-20240320.iso +gpg --verify securityonion-2.4.70-20240529.iso.sig securityonion-2.4.70-20240529.iso ``` The output should show "Good signature" and the Primary key fingerprint should match what's shown below: ``` -gpg: Signature made Tue 19 Mar 2024 03:17:58 PM EDT using RSA key ID FE507013 +gpg: Signature made Wed 29 May 2024 11:40:59 AM EDT using RSA key ID FE507013 gpg: Good signature from "Security Onion Solutions, LLC " gpg: WARNING: This key is not certified with a trusted signature! gpg: There is no indication that the signature belongs to the owner. diff --git a/sigs/securityonion-2.4.70-20240529.iso.sig b/sigs/securityonion-2.4.70-20240529.iso.sig new file mode 100644 index 0000000000000000000000000000000000000000..c3825eb6e72a860f67708c45b3926d7fdc90b650 GIT binary patch literal 566 zcmV-60?GY}0y6{v0SEvc79j-41gSkXz6^6dp_W8^5Ma0dP;e6k0%liCivS7<5PT3| zxBgIY6IO;00GUA4^4cz_ff*If5}A0oP7{X5>Z=GC&kVQad2loDR}h58qGE)=Rx^yp z-+xRh;q#;TL)Clt%_Pqf`(+ZrTTxv(K=m6LP%GfSK4WmEzCYmDGF{x`Aax-QCl14M?B(?m6}F< z8ytoSd0DJjf3dNdprN}#!)CDM%UoA1<6_PF(m7Ed~UxN<0?0bGK zYZmg%ne6j4P621BW!vQmH~#^`e1^aJc(KS3%i>sop?;h9?k z$XnSrS90)^;tYA8-5UvbUO{ZLYxgUGGhIA}2I}y$to?u$rExzvc+R*l{a!nVSe@G} zA9x@C>(y#~9Pmj<<|Xctul~dkY1TD+#W?!T-m;neL){fdtG)HlsOCQTaX0s?y^bC10U|My|kDl##27 Date: Wed, 29 May 2024 14:52:47 -0400 Subject: [PATCH 100/101] 2.4.70 --- DOWNLOAD_AND_VERIFY_ISO.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DOWNLOAD_AND_VERIFY_ISO.md b/DOWNLOAD_AND_VERIFY_ISO.md index fcefce469..a5fd6e157 100644 --- a/DOWNLOAD_AND_VERIFY_ISO.md +++ b/DOWNLOAD_AND_VERIFY_ISO.md @@ -1,4 +1,4 @@ -### 2.4.70-20240529 ISO image released on 2024/03/20 +### 2.4.70-20240529 ISO image released on 2024/05/29 ### Download and Verify From 62bdb2627aea9378f7c11ae9a72f04cbf2a14e96 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 29 May 2024 16:53:27 -0400 Subject: [PATCH 101/101] Update VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index b3c5d8c27..d2587d896 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4.70 +2.4.80