From 61ab1f1ef285c3cd79bd7307c820283683878f48 Mon Sep 17 00:00:00 2001 From: weslambert Date: Thu, 15 Aug 2024 23:03:07 -0400 Subject: [PATCH 01/32] Add tenable_io templates --- salt/elasticsearch/defaults.yaml | 184 +++++++++++++++++++++++++++++++ 1 file changed, 184 insertions(+) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index b18ab5a67..7201df25e 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -9282,6 +9282,190 @@ elasticsearch: set_priority: priority: 50 min_age: 30d + so-logs-tenable_io_x_asset: + index_sorting: False + index_template: + index_patterns: + - "logs-tenable_io.asset-*" + template: + settings: + index: + lifecycle: + name: so-logs-tenable_io.asset-logs + number_of_replicas: 0 + composed_of: + - "logs-tenable_io.asset@package" + - "logs-tenable_io.asset@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + ignore_missing_component_templates: + - logs-tenable_io.asset@custom + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-tenable_io_x_plugin: + index_sorting: False + index_template: + index_patterns: + - "logs-tenable_io.plugin-*" + template: + settings: + index: + lifecycle: + name: so-logs-tenable_io.plugin-logs + number_of_replicas: 0 + composed_of: + - "logs-tenable_io.plugin@package" + - "logs-tenable_io.plugin@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + ignore_missing_component_templates: + - logs-tenable_io.plugin@custom + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-tenable_io_x_scan: + index_sorting: False + index_template: + index_patterns: + - "logs-tenable_io.scan-*" + template: + settings: + index: + lifecycle: + name: so-logs-tenable_io.scan-logs + number_of_replicas: 0 + composed_of: + - "logs-tenable_io.scan@package" + - "logs-tenable_io.scan@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + ignore_missing_component_templates: + - logs-tenable_io.scan@custom + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-tenable_io_x_vulnerability: + index_sorting: False + index_template: + index_patterns: + - "logs-tenable_io.vulnerability-*" + template: + settings: + index: + lifecycle: + name: so-logs-tenable_io.vulnerability-logs + number_of_replicas: 0 + composed_of: + - "logs-tenable_io.vulnerability@package" + - "logs-tenable_io.vulnerability@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + ignore_missing_component_templates: + - logs-tenable_io.vulnerability@custom + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d so-logs-tenable_sc_x_asset: index_sorting: false index_template: From f182833a8df1b4e338aa8718294d85108a518534 Mon Sep 17 00:00:00 2001 From: weslambert Date: Thu, 15 Aug 2024 23:03:32 -0400 Subject: [PATCH 02/32] Add tenable_io --- salt/elasticfleet/defaults.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/elasticfleet/defaults.yaml b/salt/elasticfleet/defaults.yaml index 2d9ab97a1..48b24809e 100644 --- a/salt/elasticfleet/defaults.yaml +++ b/salt/elasticfleet/defaults.yaml @@ -97,6 +97,7 @@ elasticfleet: - symantec_endpoint - system - tcp + - tenable_io - tenable_sc - ti_abusech - ti_anomali From dc197f6a5cb028edeb60eed770df899a9b7e1453 Mon Sep 17 00:00:00 2001 From: weslambert Date: Thu, 15 Aug 2024 23:06:53 -0400 Subject: [PATCH 03/32] Add tenable settings --- salt/elasticsearch/soc_elasticsearch.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index 085aab7f0..d30706837 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -466,6 +466,13 @@ elasticsearch: so-logs-sonicwall_firewall_x_log: *indexSettings so-logs-snort_x_log: *indexSettings so-logs-symantec_endpoint_x_log: *indexSettings + so-logs-tenable_io_x_asset: *indexSettings + so-logs-tenable_io_x_plugin: *indexSettings + so-logs-tenable_io_x_scan: *indexSettings + so-logs-tenable_io_x_vulnerability: *indexSettings + so-logs-tenable_sc_x_asset: *indexSettings + so-logs-tenable_sc_x_plugin: *indexSettings + so-logs-tenable_sc_x_vulnerability: *indexSettings so-logs-ti_abusech_x_malware: *indexSettings so-logs-ti_abusech_x_malwarebazaar: *indexSettings so-logs-ti_abusech_x_threatfox: *indexSettings From 224bc6b4291de8bf36fcbebc3766933908f51b24 Mon Sep 17 00:00:00 2001 From: weslambert Date: Fri, 16 Aug 2024 14:15:10 -0400 Subject: [PATCH 04/32] Ignore old SOC logs before licenseStatus --- salt/common/tools/sbin/so-log-check | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check index 3d019fb41..c5bc4d1f2 100755 --- a/salt/common/tools/sbin/so-log-check +++ b/salt/common/tools/sbin/so-log-check @@ -206,6 +206,7 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|detect-parse" # Suricata encountering a malformed rule EXCLUDED_ERRORS="$EXCLUDED_ERRORS|integrity check failed" # Detections: Exclude false positive due to automated testing EXCLUDED_ERRORS="$EXCLUDED_ERRORS|syncErrors" # Detections: Not an actual error + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Provided Grok expressions do not match field value: [unprovisioned] # SOC log: before fields.status was changed to fields.licenseStatus fi RESULT=0 From 205bbd9c61078bd307edb77e8e663c36137e6918 Mon Sep 17 00:00:00 2001 From: weslambert Date: Fri, 16 Aug 2024 14:31:11 -0400 Subject: [PATCH 05/32] Use more specific match --- salt/common/tools/sbin/so-log-check | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check index c5bc4d1f2..8f7e29d51 100755 --- a/salt/common/tools/sbin/so-log-check +++ b/salt/common/tools/sbin/so-log-check @@ -206,7 +206,7 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|detect-parse" # Suricata encountering a malformed rule EXCLUDED_ERRORS="$EXCLUDED_ERRORS|integrity check failed" # Detections: Exclude false positive due to automated testing EXCLUDED_ERRORS="$EXCLUDED_ERRORS|syncErrors" # Detections: Not an actual error - EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Provided Grok expressions do not match field value: [unprovisioned] # SOC log: before fields.status was changed to fields.licenseStatus + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Provided Grok expressions do not match field value\\: \\[unprovisioned\\]" # SOC log: before fields.status was changed to fields.licenseStatus fi RESULT=0 From df6ff027b549bb582f0e151a72dae3ef49883392 Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Mon, 19 Aug 2024 16:05:27 -0400 Subject: [PATCH 06/32] Remove unneeded elastic upgrade config --- salt/manager/tools/sbin/soup | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 521247eeb..e634e0489 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -453,8 +453,6 @@ post_to_2.4.20() { } post_to_2.4.30() { - echo "Regenerating Elastic Agent Installers" - /sbin/so-elastic-agent-gen-installers # there is an occasional error with this state: pki_public_ca_crt: TypeError: list indices must be integers or slices, not str set +e salt-call state.apply ca queue=True @@ -479,8 +477,7 @@ post_to_2.4.50() { } post_to_2.4.60() { - echo "Regenerating Elastic Agent Installers..." - so-elastic-agent-gen-installers + echo "Nothing to apply" POSTVERSION=2.4.60 } @@ -507,7 +504,8 @@ post_to_2.4.90() { } post_to_2.4.100() { - echo "Nothing to apply" + echo "Regenerating Elastic Agent Installers" + /sbin/so-elastic-agent-gen-installers POSTVERSION=2.4.100 } @@ -587,18 +585,7 @@ up_to_2.4.20() { } up_to_2.4.30() { - - # Remove older defend integration json & installed integration - rm -f /opt/so/conf/elastic-fleet/integrations/endpoints-initial/elastic-defend-endpoints.json - - . $UPDATE_DIR/salt/elasticfleet/tools/sbin/so-elastic-fleet-common - elastic_fleet_integration_remove endpoints-initial elastic-defend-endpoints - - rm -f /opt/so/state/eaintegrations.txt - - # Elastic Update for this release, so download Elastic Agent files - determine_elastic_agent_upgrade - rm -f /opt/so/state/estemplates*.txt + echo "Nothing to do for 2.4.30" INSTALLEDVERSION=2.4.30 } From ca209ed54c180f8c207b6870316e8d8743283e69 Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Tue, 20 Aug 2024 09:14:08 -0400 Subject: [PATCH 07/32] Disable auto-upgrade --- salt/elasticfleet/enabled.sls | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/elasticfleet/enabled.sls b/salt/elasticfleet/enabled.sls index af5e552eb..fb8f41329 100644 --- a/salt/elasticfleet/enabled.sls +++ b/salt/elasticfleet/enabled.sls @@ -138,10 +138,10 @@ so-elastic-fleet-integrations: cmd.run: - name: /usr/sbin/so-elastic-fleet-integration-policy-load -so-elastic-agent-grid-upgrade: - cmd.run: - - name: /usr/sbin/so-elastic-agent-grid-upgrade - - retry: True +#so-elastic-agent-grid-upgrade: +# cmd.run: +# - name: /usr/sbin/so-elastic-agent-grid-upgrade +# - retry: True {% endif %} delete_so-elastic-fleet_so-status.disabled: From e3ecc9d4bed34044ecda62841c50e35f92c99718 Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Tue, 20 Aug 2024 15:06:16 -0400 Subject: [PATCH 08/32] Directly manage the Fleet Server integration config --- .../fleet-server/fleet-server.json | 21 +++++++++++++++++++ .../tools/sbin_jinja/so-elastic-fleet-setup | 5 ++++- 2 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 salt/elasticfleet/files/integrations-dynamic/fleet-server/fleet-server.json diff --git a/salt/elasticfleet/files/integrations-dynamic/fleet-server/fleet-server.json b/salt/elasticfleet/files/integrations-dynamic/fleet-server/fleet-server.json new file mode 100644 index 000000000..202345c80 --- /dev/null +++ b/salt/elasticfleet/files/integrations-dynamic/fleet-server/fleet-server.json @@ -0,0 +1,21 @@ +{% from 'vars/globals.map.jinja' import GLOBALS %} + +{ + "package": { + "name": "fleet_server", + "version": "" + }, + "name": "fleet_server-1", + "namespace": "default", + "policy_id": "FleetServer_{{ GLOBALS.hostname }}", + "vars": {}, + "inputs": { + "fleet_server-fleet-server": { + "enabled": true, + "vars": { + "custom": "server.ssl.supported_protocols: [\"TLSv1.2\", \"TLSv1.3\"]\nserver.ssl.cipher_suites: [ \"ECDHE-RSA-AES-128-GCM-SHA256\", \"ECDHE-RSA-AES-256-GCM-SHA384\", \"ECDHE-RSA-AES-128-CBC-SHA\", \"ECDHE-RSA-AES-256-CBC-SHA\", \"RSA-AES-128-GCM-SHA256\", \"RSA-AES-256-GCM-SHA384\"]" + }, + "streams": {} + } + } +} diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup index d53a5a21a..714c2d407 100755 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup @@ -66,7 +66,10 @@ printf "\n\n" # Create the Manager Fleet Server Host Agent Policy # This has to be done while the Elasticsearch Output is set to the default Output printf "Create Manager Fleet Server Policy...\n" -elastic_fleet_policy_create "FleetServer_{{ GLOBALS.hostname }}" "Fleet Server - {{ GLOBALS.hostname }}" "true" "120" +elastic_fleet_policy_create "FleetServer_{{ GLOBALS.hostname }}" "Fleet Server - {{ GLOBALS.hostname }}" "false" "120" + +# Now that the Manager Fleet Server Policy is created, add the Fleet Server Integration to it +elastic_fleet_integration_create "@/opt/so/conf/elastic-fleet/integrations/fleet-server/fleet-server.json" # Now we can create the Logstash Output and set it to to be the default Output printf "\n\nCreate Logstash Output Config if node is not an Import or Eval install\n" From 07f8bda27e26a3e44e3493524a74d372dce98e66 Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Tue, 20 Aug 2024 15:23:31 -0400 Subject: [PATCH 09/32] Update agent --- salt/elasticfleet/enabled.sls | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/elasticfleet/enabled.sls b/salt/elasticfleet/enabled.sls index fb8f41329..af5e552eb 100644 --- a/salt/elasticfleet/enabled.sls +++ b/salt/elasticfleet/enabled.sls @@ -138,10 +138,10 @@ so-elastic-fleet-integrations: cmd.run: - name: /usr/sbin/so-elastic-fleet-integration-policy-load -#so-elastic-agent-grid-upgrade: -# cmd.run: -# - name: /usr/sbin/so-elastic-agent-grid-upgrade -# - retry: True +so-elastic-agent-grid-upgrade: + cmd.run: + - name: /usr/sbin/so-elastic-agent-grid-upgrade + - retry: True {% endif %} delete_so-elastic-fleet_so-status.disabled: From f01825166d22533ca9620d936b1190ed6ccbc224 Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Wed, 21 Aug 2024 08:31:37 -0400 Subject: [PATCH 10/32] Update Fleet Server policy --- ...et-integration-policy-elastic-fleet-server | 23 +++++++++++++++++++ .../so-elastic-fleet-integration-policy-load | 5 +++- 2 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-elastic-fleet-server diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-elastic-fleet-server b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-elastic-fleet-server new file mode 100644 index 000000000..42ffaede2 --- /dev/null +++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-elastic-fleet-server @@ -0,0 +1,23 @@ +#!/bin/bash +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +. /usr/sbin/so-elastic-fleet-common + +# Make the curl request to fetch the JSON data +json_output=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -L -X GET "localhost:5601/api/fleet/agent_policies" -H 'kbn-xsrf: true') + +# Extract the IDs that start with "FleetServer_" using jq +POLICY=$(echo "$json_output" | jq -r '.items[] | select(.id | startswith("FleetServer_")) | .id') + +echo $POLICY + +# Iterate over each ID in the POLICY variable and run the specified commands +for POLICYNAME in $POLICY; do + # First get the Integration ID + elastic_fleet_integration_check "$POLICYNAME" "/opt/so/conf/elastic-fleet/integrations/fleet-server/fleet-server.json" + # Now update the integration policy + elastic_fleet_integration_update "$INTEGRATION_ID" "@/opt/so/conf/elastic-fleet/integrations/fleet-server/fleet-server.json" +done \ No newline at end of file diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load index 518d29d26..26414a94b 100644 --- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load +++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load @@ -12,7 +12,10 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then # First, check for any package upgrades /usr/sbin/so-elastic-fleet-package-upgrade - # Second, configure Elastic Defend Integration seperately + # Second, update Fleet Server policies + /sbin/so-elastic-fleet-integration-policy-elastic-fleet-server + + # Third, configure Elastic Defend Integration seperately /usr/sbin/so-elastic-fleet-integration-policy-elastic-defend # Initial Endpoints From c30537fe6a9f8bff5cbdca0304f24f0aa418372d Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 21 Aug 2024 13:00:04 -0400 Subject: [PATCH 11/32] Ensure endpoint is installed --- .../tools/sbin_jinja/so-elasticsearch-templates-load | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load index 12ef4dbf6..5d1e88798 100755 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load @@ -69,7 +69,7 @@ if [ ! -f $STATE_FILE_SUCCESS ]; then retry 240 1 "so-elasticsearch-query / -k --output /dev/null --silent --head --fail" || fail "Connection attempt timed out. Unable to connect to ElasticSearch. \nPlease try: \n -checking log(s) in /var/log/elasticsearch/\n -running 'sudo docker ps' \n -running 'sudo so-elastic-restart'" {% if GLOBALS.role != 'so-heavynode' %} SESSIONCOOKIE=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}') - INSTALLED=$(elastic_fleet_package_is_installed {{ SUPPORTED_PACKAGES[0] }} ) + INSTALLED=$(elastic_fleet_package_is_installed endpoint }} ) if [ "$INSTALLED" != "installed" ]; then echo echo "Packages not yet installed." From 4afac201b93f0502573a574184774e84afb667a9 Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 21 Aug 2024 13:25:26 -0400 Subject: [PATCH 12/32] Change ILM policy name --- salt/elasticsearch/defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 7201df25e..be490842f 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -3679,7 +3679,7 @@ elasticsearch: settings: index: lifecycle: - name: so-logs-detections.alerts-so + name: so-logs-detections.alerts-logs mapping: total_fields: limit: 5001 From ff479de7bdc26407cebafb1039457553b039e3a2 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 21 Aug 2024 14:10:24 -0400 Subject: [PATCH 13/32] Add support for new appliance raid controllers --- salt/common/tools/sbin_jinja/so-raid-status | 67 ++++++++++++++------- 1 file changed, 44 insertions(+), 23 deletions(-) diff --git a/salt/common/tools/sbin_jinja/so-raid-status b/salt/common/tools/sbin_jinja/so-raid-status index 6cd8b84de..3fe238c23 100755 --- a/salt/common/tools/sbin_jinja/so-raid-status +++ b/salt/common/tools/sbin_jinja/so-raid-status @@ -9,6 +9,9 @@ . /usr/sbin/so-common +software_raid=("SOSMN" "SOSMN-DE02" "SOSSNNV" "SOSSNNV-DE02" "SOS10k-DE02" "SOS10KNV" "SOS10KNV-DE02" "SOS10KNV-DE02" "SOS2000-DE02" "SOS-GOFAST-LT-DE02" "SOS-GOFAST-MD-DE02" "SOS-GOFAST-HV-DE02") +hardware_raid=("SOS1000" "SOS1000F" "SOSSN7200" "SOS5000" "SOS4000") + {%- if salt['grains.get']('sosmodel', '') %} {%- set model = salt['grains.get']('sosmodel') %} model={{ model }} @@ -16,33 +19,42 @@ model={{ model }} if [[ $model =~ ^(SO2AMI01|SO2AZI01|SO2GCI01)$ ]]; then exit 0 fi + +for i in "${software_raid[@]}"; do + if [[ "$model" == $i ]]; then + is_softwareraid=true + is_hwraid=false + break + fi +done + +for i in "${hardware_raid[@]}"; do + if [[ "$model" == $i ]]; then + is_softwareraid=false + is_hwraid=true + break + fi +done + {%- else %} echo "This is not an appliance" exit 0 {%- endif %} -if [[ $model =~ ^(SOS10K|SOS500|SOS1000|SOS1000F|SOS4000|SOSSN7200|SOSSNNV|SOSMN)$ ]]; then - is_bossraid=true -fi -if [[ $model =~ ^(SOSSNNV|SOSMN)$ ]]; then - is_swraid=true -fi -if [[ $model =~ ^(SOS10K|SOS500|SOS1000|SOS1000F|SOS4000|SOSSN7200)$ ]]; then - is_hwraid=true -fi check_nsm_raid() { PERCCLI=$(/opt/raidtools/perccli/perccli64 /c0/v0 show|grep RAID|grep Optl) MEGACTL=$(/opt/raidtools/megasasctl |grep optimal) - - if [[ $APPLIANCE == '1' ]]; then + if [[ "$model" == "SOS500" || "$model" == "SOS500-DE02" ]]; then + #This doesn't have raid + HWRAID=0 + else if [[ -n $PERCCLI ]]; then HWRAID=0 elif [[ -n $MEGACTL ]]; then HWRAID=0 else HWRAID=1 - fi - + fi fi } @@ -50,17 +62,27 @@ check_nsm_raid() { check_boss_raid() { MVCLI=$(/usr/local/bin/mvcli info -o vd |grep status |grep functional) MVTEST=$(/usr/local/bin/mvcli info -o vd | grep "No adapter") + BOSSNVMECLI=$(/usr/local/bin/mnv_cli info -o vd -i 0 | grep Functional) - # Check to see if this is a SM based system - if [[ -z $MVTEST ]]; then - if [[ -n $MVCLI ]]; then + # Is this NVMe Boss Raid? + if [[ "$model" =~ "-DE02" ]]; then + if [[ -n $BOSSNVMECLI ]]; then BOSSRAID=0 else BOSSRAID=1 fi else - # This doesn't have boss raid so lets make it 0 - BOSSRAID=0 + # Check to see if this is a SM based system + if [[ -z $MVTEST ]]; then + if [[ -n $MVCLI ]]; then + BOSSRAID=0 + else + BOSSRAID=1 + fi + else + # This doesn't have boss raid so lets make it 0 + BOSSRAID=0 + fi fi } @@ -79,14 +101,13 @@ SWRAID=0 BOSSRAID=0 HWRAID=0 -if [[ $is_hwraid ]]; then +if [[ "$is_hwraid" == "true" ]]; then check_nsm_raid + check_boss_raid fi -if [[ $is_bossraid ]]; then - check_boss_raid -fi -if [[ $is_swraid ]]; then +if [[ "$is_softwareraid" == "true" ]]; then check_software_raid + check_boss_raid fi sum=$(($SWRAID + $BOSSRAID + $HWRAID)) From 4108e6717880d549a80443e22491c78b7acbff40 Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 21 Aug 2024 14:22:28 -0400 Subject: [PATCH 14/32] Check for endpoint package --- .../tools/sbin_jinja/so-elasticsearch-templates-load | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load index 12ef4dbf6..381e33fe4 100755 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load @@ -5,7 +5,6 @@ # Elastic License 2.0. {%- import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %} {% from 'vars/globals.map.jinja' import GLOBALS %} -{%- set SUPPORTED_PACKAGES = salt['pillar.get']('elasticfleet:packages', default=ELASTICFLEETDEFAULTS.elasticfleet.packages, merge=True) %} STATE_FILE_INITIAL=/opt/so/state/estemplates_initial_load_attempt.txt STATE_FILE_SUCCESS=/opt/so/state/estemplates.txt @@ -69,7 +68,7 @@ if [ ! -f $STATE_FILE_SUCCESS ]; then retry 240 1 "so-elasticsearch-query / -k --output /dev/null --silent --head --fail" || fail "Connection attempt timed out. Unable to connect to ElasticSearch. \nPlease try: \n -checking log(s) in /var/log/elasticsearch/\n -running 'sudo docker ps' \n -running 'sudo so-elastic-restart'" {% if GLOBALS.role != 'so-heavynode' %} SESSIONCOOKIE=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}') - INSTALLED=$(elastic_fleet_package_is_installed {{ SUPPORTED_PACKAGES[0] }} ) + INSTALLED=$(elastic_fleet_package_is_installed endpoint }} ) if [ "$INSTALLED" != "installed" ]; then echo echo "Packages not yet installed." From c1b7232a883d1a0fd654b7dd43c50e9ac49191f5 Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 21 Aug 2024 14:38:29 -0400 Subject: [PATCH 15/32] Fix for detections-alerts --- .../sbin_jinja/so-elasticsearch-ilm-policy-load | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load index b00fcbedf..77178b4fe 100755 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load @@ -10,10 +10,16 @@ {%- for index, settings in ES_INDEX_SETTINGS.items() %} {%- if settings.policy is defined %} +{%- if index == 'so-logs-detections.alerts' %} echo -echo "Setting up {{ index }}-logs policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://localhost:9200/_ilm/policy/{{ index }}-logs" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }' -echo + echo "Setting up so-logs-detections-alerts-so policy..." + curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://localhost:9200/_ilm/policy/{{ index }}-so" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }' + echo +{%- else %} + echo "Setting up {{ index }}-logs policy..." + curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://localhost:9200/_ilm/policy/{{ index }}-logs" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }' + echo +{%- endif %} {%- endif %} {%- endfor %} echo From 88ea60df2ae02ae13e9182b86b67d4aa86de6cb9 Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 21 Aug 2024 14:38:57 -0400 Subject: [PATCH 16/32] Fix name --- .../tools/sbin_jinja/so-elasticsearch-ilm-policy-load | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load index 77178b4fe..7d3894950 100755 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load @@ -12,7 +12,7 @@ {%- if settings.policy is defined %} {%- if index == 'so-logs-detections.alerts' %} echo - echo "Setting up so-logs-detections-alerts-so policy..." + echo "Setting up so-logs-detections.alerts-so policy..." curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://localhost:9200/_ilm/policy/{{ index }}-so" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }' echo {%- else %} From 212cc478dea31d9be4aaaadea7f6984db12334c7 Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 21 Aug 2024 14:39:24 -0400 Subject: [PATCH 17/32] Change back to so --- salt/elasticsearch/defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index be490842f..7201df25e 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -3679,7 +3679,7 @@ elasticsearch: settings: index: lifecycle: - name: so-logs-detections.alerts-logs + name: so-logs-detections.alerts-so mapping: total_fields: limit: 5001 From cf475081853c6380bed1683899a8709913191f01 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 22 Aug 2024 09:02:32 -0400 Subject: [PATCH 18/32] notification updates --- salt/elastalert/soc_elastalert.yaml | 4 +- salt/soc/soc_soc.yaml | 89 ++++++++++++++++++++++++++++- 2 files changed, 88 insertions(+), 5 deletions(-) diff --git a/salt/elastalert/soc_elastalert.yaml b/salt/elastalert/soc_elastalert.yaml index 435c5be6a..905fd3884 100644 --- a/salt/elastalert/soc_elastalert.yaml +++ b/salt/elastalert/soc_elastalert.yaml @@ -3,8 +3,8 @@ elastalert: description: You can enable or disable Elastalert. helpLink: elastalert.html alerter_parameters: - title: Alerter Parameters - description: Optional configuration parameters for additional alerters that can be enabled for all Sigma rules. Filter for 'Alerter' in this Configuration screen to find the setting that allows these alerters to be enabled within the SOC ElastAlert module. Use YAML format for these parameters, and reference the ElastAlert 2 documentation, located at https://elastalert2.readthedocs.io, for available alerters and their required configuration parameters. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key. + title: Custom Configuration Parameters + description: Optional configuration parameters made available as defaults for all rules and alerters. Use YAML format for these parameters, and reference the ElastAlert 2 documentation, located at https://elastalert2.readthedocs.io, for available configuration parameters. Requires a valid Security Onion license key. global: True multiline: True syntax: yaml diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index d2f63e4ad..d82c32459 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -103,12 +103,95 @@ soc: description: Show AI summaries for ElastAlert rules. global: True additionalAlerters: - title: Additional Alerters - description: Specify additional alerters to enable for all Sigma rules, one alerter name per line. Alerters refers to ElastAlert 2 alerters, as documented at https://elastalert2.readthedocs.io. Note that the configuration parameters for these alerters must be provided in the ElastAlert configuration section. Filter for 'Alerter' to find this related setting. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key. + title: "Notifications: Sev 0/Default Alerters" + description: "Specify default alerters to enable for outbound notifications. These alerters will be used unless overriden by higher severity alerter settings. Specify one alerter name (Ex: 'email') per line. Alerters refers to ElastAlert 2 alerters, as documented at https://elastalert2.readthedocs.io. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key." global: True - helpLink: sigma.html + helpLink: notifications.html forcedType: "[]string" multiline: True + additionalSev0AlertersParams: + title: "Notifications: Sev 0/Default Parameters" + description: Optional configuration parameters for default alerters. Use YAML format for these parameters, and reference the ElastAlert 2 documentation, located at https://elastalert2.readthedocs.io, for available alerters and their required configuration parameters. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key. + global: True + multiline: True + syntax: yaml + helpLink: notifications.html + forcedType: string + additionalSev1Alerters: + title: "Notifications: Sev 1/Informational Alerters" + description: "Specify specific alerters to use when alerting at the info severity level or higher. These alerters will be used unless overriden by higher severity alerter settings. Specify one alerter name (Ex: 'email') per line. Alerters refers to ElastAlert 2 alerters, as documented at https://elastalert2.readthedocs.io. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key." + global: True + helpLink: notifications.html + forcedType: "[]string" + multiline: True + additionalSev1AlertersParams: + title: "Notifications: Sev 1/Informational Parameters" + description: Optional configuration parameters for informational severity alerters. Info level is less severe than 'Low Severity'. Use YAML format for these parameters, and reference the ElastAlert 2 documentation, located at https://elastalert2.readthedocs.io, for available alerters and their required configuration parameters. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key. + global: True + multiline: True + syntax: yaml + helpLink: notifications.html + forcedType: string + additionalSev2Alerters: + title: "Notifications: Sev 2/Low Alerters" + description: "Specify specific alerters to use when alerting at the low severity level or higher. These alerters will be used unless overriden by higher severity alerter settings. Specify one alerter name (Ex: 'email') per line. Alerters refers to ElastAlert 2 alerters, as documented at https://elastalert2.readthedocs.io. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key." + global: True + helpLink: notifications.html + forcedType: "[]string" + multiline: True + additionalSev2AlertersParams: + title: "Notifications: Sev 2/Low Parameters" + description: Optional configuration parameters for low severity alerters. Use YAML format for these parameters, and reference the ElastAlert 2 documentation, located at https://elastalert2.readthedocs.io, for available alerters and their required configuration parameters. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key. + global: True + multiline: True + syntax: yaml + helpLink: notifications.html + forcedType: string + additionalSev3Alerters: + title: "Notifications: Sev 3/Medium Alerters" + description: "Specify specific alerters to use when alerting at the medium severity level or higher. These alerters will be used unless overriden by higher severity alerter settings. Specify one alerter name (Ex: 'email') per line. Alerters refers to ElastAlert 2 alerters, as documented at https://elastalert2.readthedocs.io. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key." + global: True + helpLink: notifications.html + forcedType: "[]string" + multiline: True + additionalSev3AlertersParams: + title: "Notifications: Sev 3/Medium Parameters" + description: Optional configuration parameters for medium severity alerters. Use YAML format for these parameters, and reference the ElastAlert 2 documentation, located at https://elastalert2.readthedocs.io, for available alerters and their required configuration parameters. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key. + global: True + multiline: True + syntax: yaml + helpLink: notifications.html + forcedType: string + additionalSev4Alerters: + title: "Notifications: Sev 4/High Alerters" + description: "Specify specific alerters to use when alerting at the high severity level or critical severity level. These alerters will be used unless overriden by critical severity alerter settings. Specify one alerter name (Ex: 'email') per line. Alerters refers to ElastAlert 2 alerters, as documented at https://elastalert2.readthedocs.io. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key." + global: True + helpLink: notifications.html + forcedType: "[]string" + multiline: True + additionalSev4AlertersParams: + title: "Notifications: Sev 4/High Parameters" + description: Optional configuration parameters for high severity alerters. Use YAML format for these parameters, and reference the ElastAlert 2 documentation, located at https://elastalert2.readthedocs.io, for available alerters and their required configuration parameters. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key. + global: True + multiline: True + syntax: yaml + helpLink: notifications.html + forcedType: string + additionalSev5Alerters: + title: "Notifications: Sev 5/Critical Alerters" + description: "Specify specific alerters to use when alerting at the critical severity level. Specify one alerter name (Ex: 'email') per line. Alerters refers to ElastAlert 2 alerters, as documented at https://elastalert2.readthedocs.io. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key." + global: True + helpLink: notifications.html + forcedType: "[]string" + multiline: True + additionalSev5AlertersParams: + title: "Notifications: Sev 5/Critical Parameters" + description: Optional configuration parameters for critical severity alerters. Use YAML format for these parameters, and reference the ElastAlert 2 documentation, located at https://elastalert2.readthedocs.io, for available alerters and their required configuration parameters. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key. + global: True + multiline: True + syntax: yaml + helpLink: notifications.html + forcedType: string autoEnabledSigmaRules: default: &autoEnabledSigmaRules description: 'Sigma rules to automatically enable on initial import. Format is $Ruleset+$Level - for example, for the core community ruleset and critical level rules: core+critical. These will be applied based on role if defined and default if not.' From 48f1e24bf52650fbf8f015228e9bb4a6a93fdc2e Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 22 Aug 2024 09:04:43 -0400 Subject: [PATCH 19/32] notification updates --- salt/soc/soc_soc.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index d82c32459..ff7f8efd0 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -104,7 +104,7 @@ soc: global: True additionalAlerters: title: "Notifications: Sev 0/Default Alerters" - description: "Specify default alerters to enable for outbound notifications. These alerters will be used unless overriden by higher severity alerter settings. Specify one alerter name (Ex: 'email') per line. Alerters refers to ElastAlert 2 alerters, as documented at https://elastalert2.readthedocs.io. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key." + description: "Specify default alerters to enable for outbound notifications. These alerters will be used unless overridden by higher severity alerter settings. Specify one alerter name (Ex: 'email') per line. Alerters refers to ElastAlert 2 alerters, as documented at https://elastalert2.readthedocs.io. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key." global: True helpLink: notifications.html forcedType: "[]string" @@ -119,7 +119,7 @@ soc: forcedType: string additionalSev1Alerters: title: "Notifications: Sev 1/Informational Alerters" - description: "Specify specific alerters to use when alerting at the info severity level or higher. These alerters will be used unless overriden by higher severity alerter settings. Specify one alerter name (Ex: 'email') per line. Alerters refers to ElastAlert 2 alerters, as documented at https://elastalert2.readthedocs.io. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key." + description: "Specify specific alerters to use when alerting at the info severity level or higher. These alerters will be used unless overridden by higher severity alerter settings. Specify one alerter name (Ex: 'email') per line. Alerters refers to ElastAlert 2 alerters, as documented at https://elastalert2.readthedocs.io. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key." global: True helpLink: notifications.html forcedType: "[]string" @@ -134,7 +134,7 @@ soc: forcedType: string additionalSev2Alerters: title: "Notifications: Sev 2/Low Alerters" - description: "Specify specific alerters to use when alerting at the low severity level or higher. These alerters will be used unless overriden by higher severity alerter settings. Specify one alerter name (Ex: 'email') per line. Alerters refers to ElastAlert 2 alerters, as documented at https://elastalert2.readthedocs.io. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key." + description: "Specify specific alerters to use when alerting at the low severity level or higher. These alerters will be used unless overridden by higher severity alerter settings. Specify one alerter name (Ex: 'email') per line. Alerters refers to ElastAlert 2 alerters, as documented at https://elastalert2.readthedocs.io. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key." global: True helpLink: notifications.html forcedType: "[]string" @@ -149,7 +149,7 @@ soc: forcedType: string additionalSev3Alerters: title: "Notifications: Sev 3/Medium Alerters" - description: "Specify specific alerters to use when alerting at the medium severity level or higher. These alerters will be used unless overriden by higher severity alerter settings. Specify one alerter name (Ex: 'email') per line. Alerters refers to ElastAlert 2 alerters, as documented at https://elastalert2.readthedocs.io. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key." + description: "Specify specific alerters to use when alerting at the medium severity level or higher. These alerters will be used unless overridden by higher severity alerter settings. Specify one alerter name (Ex: 'email') per line. Alerters refers to ElastAlert 2 alerters, as documented at https://elastalert2.readthedocs.io. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key." global: True helpLink: notifications.html forcedType: "[]string" @@ -164,7 +164,7 @@ soc: forcedType: string additionalSev4Alerters: title: "Notifications: Sev 4/High Alerters" - description: "Specify specific alerters to use when alerting at the high severity level or critical severity level. These alerters will be used unless overriden by critical severity alerter settings. Specify one alerter name (Ex: 'email') per line. Alerters refers to ElastAlert 2 alerters, as documented at https://elastalert2.readthedocs.io. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key." + description: "Specify specific alerters to use when alerting at the high severity level or critical severity level. These alerters will be used unless overridden by critical severity alerter settings. Specify one alerter name (Ex: 'email') per line. Alerters refers to ElastAlert 2 alerters, as documented at https://elastalert2.readthedocs.io. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key." global: True helpLink: notifications.html forcedType: "[]string" From d7e3e134a58b2057792f5d555c823f697eabd9a7 Mon Sep 17 00:00:00 2001 From: weslambert Date: Thu, 22 Aug 2024 10:33:13 -0400 Subject: [PATCH 20/32] Check Elasticsearch for template --- .../tools/sbin_jinja/so-elasticsearch-templates-load | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load index 381e33fe4..76b1cc193 100755 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load @@ -67,9 +67,9 @@ if [ ! -f $STATE_FILE_SUCCESS ]; then echo -n "Waiting for ElasticSearch..." retry 240 1 "so-elasticsearch-query / -k --output /dev/null --silent --head --fail" || fail "Connection attempt timed out. Unable to connect to ElasticSearch. \nPlease try: \n -checking log(s) in /var/log/elasticsearch/\n -running 'sudo docker ps' \n -running 'sudo so-elastic-restart'" {% if GLOBALS.role != 'so-heavynode' %} - SESSIONCOOKIE=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}') - INSTALLED=$(elastic_fleet_package_is_installed endpoint }} ) - if [ "$INSTALLED" != "installed" ]; then + TEMPLATE="logs-endpoint.alerts@package" + INSTALLED=$(so-elasticsearch-query _component_template/$TEMPLATE | jq -r .component_templates[0].name) + if [ "$INSTALLED" != "$TEMPLATE" ]; then echo echo "Packages not yet installed." echo From eabb894580805c51f71dd9a37b9a9b8495a8a06c Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 22 Aug 2024 17:52:37 -0400 Subject: [PATCH 21/32] exclude all logstash errors related to license manager init log line --- salt/common/tools/sbin/so-log-check | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check index 8f7e29d51..25ca4721f 100755 --- a/salt/common/tools/sbin/so-log-check +++ b/salt/common/tools/sbin/so-log-check @@ -206,7 +206,7 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|detect-parse" # Suricata encountering a malformed rule EXCLUDED_ERRORS="$EXCLUDED_ERRORS|integrity check failed" # Detections: Exclude false positive due to automated testing EXCLUDED_ERRORS="$EXCLUDED_ERRORS|syncErrors" # Detections: Not an actual error - EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Provided Grok expressions do not match field value\\: \\[unprovisioned\\]" # SOC log: before fields.status was changed to fields.licenseStatus + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Initialized license manager" # SOC log: before fields.status was changed to fields.licenseStatus fi RESULT=0 From 1ec5e3bf2a43961546f215c735ab3c7c3f0af79a Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Fri, 23 Aug 2024 09:47:21 -0400 Subject: [PATCH 22/32] add kafka.id to common ingest pipeline Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/elasticsearch/files/ingest-dynamic/common | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/elasticsearch/files/ingest-dynamic/common b/salt/elasticsearch/files/ingest-dynamic/common index 836b8d4af..93e7f1cca 100644 --- a/salt/elasticsearch/files/ingest-dynamic/common +++ b/salt/elasticsearch/files/ingest-dynamic/common @@ -62,6 +62,7 @@ { "split": { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "separator": "\\.", "target_field": "dataset_tag_temp" } }, { "append": { "if": "ctx.dataset_tag_temp != null", "field": "tags", "value": "{{dataset_tag_temp.1}}" } }, { "grok": { "if": "ctx.http?.response?.status_code != null", "field": "http.response.status_code", "patterns": ["%{NUMBER:http.response.status_code:long} %{GREEDYDATA}"]} }, + { "set": { "if": "ctx?.metadata?.kafka != null" , "field": "kafka.id", "value": "{{metadata.kafka.partition}}{{metadata.kafka.offset}}{{metadata.kafka.timestamp}}", "ignore_failure": true } }, { "remove": { "field": [ "message2", "type", "fields", "category", "module", "dataset", "dataset_tag_temp", "event.dataset_temp" ], "ignore_missing": true, "ignore_failure": true } } {%- endraw %} {%- if HIGHLANDER %} From 0a5725a62ee94ccc9ef0dddaf43cfd38e45fde5c Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Fri, 23 Aug 2024 11:36:47 -0400 Subject: [PATCH 23/32] Refactor for Elastic Upgrade --- .../fleet-server/fleet-server.json | 4 +- ...et-integration-policy-elastic-fleet-server | 20 ++++++---- salt/manager/tools/sbin/so-minion | 40 ++++++++++++------- 3 files changed, 40 insertions(+), 24 deletions(-) diff --git a/salt/elasticfleet/files/integrations-dynamic/fleet-server/fleet-server.json b/salt/elasticfleet/files/integrations-dynamic/fleet-server/fleet-server.json index 202345c80..116f78500 100644 --- a/salt/elasticfleet/files/integrations-dynamic/fleet-server/fleet-server.json +++ b/salt/elasticfleet/files/integrations-dynamic/fleet-server/fleet-server.json @@ -1,5 +1,3 @@ -{% from 'vars/globals.map.jinja' import GLOBALS %} - { "package": { "name": "fleet_server", @@ -7,7 +5,7 @@ }, "name": "fleet_server-1", "namespace": "default", - "policy_id": "FleetServer_{{ GLOBALS.hostname }}", + "policy_id": "FleetServer_hostname", "vars": {}, "inputs": { "fleet_server-fleet-server": { diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-elastic-fleet-server b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-elastic-fleet-server index 42ffaede2..c304d5ba5 100644 --- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-elastic-fleet-server +++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-elastic-fleet-server @@ -6,18 +6,24 @@ . /usr/sbin/so-elastic-fleet-common -# Make the curl request to fetch the JSON data +# Get all the fleet policies json_output=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -L -X GET "localhost:5601/api/fleet/agent_policies" -H 'kbn-xsrf: true') -# Extract the IDs that start with "FleetServer_" using jq +# Extract the IDs that start with "FleetServer_" POLICY=$(echo "$json_output" | jq -r '.items[] | select(.id | startswith("FleetServer_")) | .id') -echo $POLICY - -# Iterate over each ID in the POLICY variable and run the specified commands +# Iterate over each ID in the POLICY variable for POLICYNAME in $POLICY; do + printf "\nUpdating Policy: $POLICYNAME\n" + # First get the Integration ID elastic_fleet_integration_check "$POLICYNAME" "/opt/so/conf/elastic-fleet/integrations/fleet-server/fleet-server.json" - # Now update the integration policy - elastic_fleet_integration_update "$INTEGRATION_ID" "@/opt/so/conf/elastic-fleet/integrations/fleet-server/fleet-server.json" + + # Modify the default integration policy to update the policy_id and an with the correct naming + UPDATED_INTEGRATION_POLICY=$(jq --arg policy_id "$POLICYNAME" --arg name "fleet_server-$POLICYNAME" ' + .policy_id = $policy_id | + .name = $name' /opt/so/conf/elastic-fleet/integrations/fleet-server/fleet-server.json) + + # Now update the integration policy using the modified JSON + elastic_fleet_integration_update "$INTEGRATION_ID" "$UPDATED_INTEGRATION_POLICY" done \ No newline at end of file diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index 6f14104c3..ebbfa8fff 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -9,6 +9,10 @@ if [ -f /usr/sbin/so-common ]; then . /usr/sbin/so-common fi +if [ -f /usr/sbin/so-elastic-fleet-common ]; then + . /usr/sbin/so-elastic-fleet-common +fi + function usage() { echo "Usage: $0 -o= -m=[id]" echo "" @@ -380,23 +384,31 @@ function add_elastic_fleet_package_registry_to_minion() { function create_fleet_policy() { - JSON_STRING=$( jq -n \ - --arg NAME "FleetServer_$LSHOSTNAME" \ - --arg DESC "Fleet Server - $LSHOSTNAME" \ - '{"name": $NAME,"id":$NAME,"description":$DESC,"namespace":"default","monitoring_enabled":["logs"],"inactivity_timeout":1209600,"has_fleet_server":true}' - ) + # First, set the default output to Elasticsearch + # This is required because of the license output bug + JSON_STRING=$(jq -n \ + '{ + "name": "so-manager_elasticsearch", + "type": "elasticsearch", + "is_default": true, + "is_default_monitoring": false + }') - # Create Fleet Sever Policy - curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/agent_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" + curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/outputs/so-manager_elasticsearch" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" - JSON_STRING_UPDATE=$( jq -n \ - --arg NAME "FleetServer_$LSHOSTNAME" \ - --arg DESC "Fleet Server - $LSHOSTNAME" \ - '{"name":$NAME,"description":$DESC,"namespace":"default","monitoring_enabled":["logs"],"inactivity_timeout":120,"data_output_id":"so-manager_elasticsearch"}' - ) + # Create the Fleet Server Policy + elastic_fleet_policy_create "FleetServer_$LSHOSTNAME" "Fleet Server - $LSHOSTNAME" "false" "120" - # Update Fleet Policy - ES Output - curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/agent_policies/FleetServer_$LSHOSTNAME" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING_UPDATE" + # Modify the default integration policy to update the policy_id with the correct naming + UPDATED_INTEGRATION_POLICY=$(jq --arg policy_id "FleetServer_$LSHOSTNAME" --arg name "fleet_server-$LSHOSTNAME" ' + .policy_id = $policy_id | + .name = $name' /opt/so/conf/elastic-fleet/integrations/fleet-server/fleet-server.json) + + # Add the Fleet Server Integration to the new Fleet Policy + elastic_fleet_integration_create "$UPDATED_INTEGRATION_POLICY" + + # Set the default output back to the default + /sbin/so-elastic-fleet-outputs-update } function update_fleet_host_urls() { From e96a0108c3096365191ad9aefe6805c9ccb09395 Mon Sep 17 00:00:00 2001 From: weslambert Date: Fri, 23 Aug 2024 13:05:34 -0400 Subject: [PATCH 24/32] Add global@custom --- salt/elasticsearch/files/ingest-dynamic/common | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/salt/elasticsearch/files/ingest-dynamic/common b/salt/elasticsearch/files/ingest-dynamic/common index 93e7f1cca..e84702909 100644 --- a/salt/elasticsearch/files/ingest-dynamic/common +++ b/salt/elasticsearch/files/ingest-dynamic/common @@ -73,7 +73,9 @@ } } {%- endif %} -{%- raw %} +{%- raw %} + , + { "pipeline": { "name": "global@custom", "ignore_missing_pipeline": true, "description": "[Fleet] Global pipeline for all data streams" } } ] } {% endraw %} From c575e02fbb344b2081540a2459c725e7d7658126 Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Fri, 23 Aug 2024 13:52:20 -0400 Subject: [PATCH 25/32] Use correct name --- .../elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup index 714c2d407..deb16dadf 100755 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup @@ -68,8 +68,13 @@ printf "\n\n" printf "Create Manager Fleet Server Policy...\n" elastic_fleet_policy_create "FleetServer_{{ GLOBALS.hostname }}" "Fleet Server - {{ GLOBALS.hostname }}" "false" "120" -# Now that the Manager Fleet Server Policy is created, add the Fleet Server Integration to it -elastic_fleet_integration_create "@/opt/so/conf/elastic-fleet/integrations/fleet-server/fleet-server.json" +# Modify the default integration policy to update the policy_id with the correct naming +UPDATED_INTEGRATION_POLICY=$(jq --arg policy_id "FleetServer_{{ GLOBALS.hostname }}" --arg name "fleet_server-{{ GLOBALS.hostname }}" ' +.policy_id = $policy_id | +.name = $name' /opt/so/conf/elastic-fleet/integrations/fleet-server/fleet-server.json) + +# Add the Fleet Server Integration to the new Fleet Policy +elastic_fleet_integration_create "$UPDATED_INTEGRATION_POLICY" # Now we can create the Logstash Output and set it to to be the default Output printf "\n\nCreate Logstash Output Config if node is not an Import or Eval install\n" From dd09f5b153a615b8d086f1cd39e85048d1d37b1d Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 26 Aug 2024 10:32:27 -0400 Subject: [PATCH 26/32] Add so-soc-logs --- .../sbin_jinja/so-elasticsearch-ilm-policy-load | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load index 7d3894950..04a7a8ab0 100755 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load @@ -11,11 +11,21 @@ {%- for index, settings in ES_INDEX_SETTINGS.items() %} {%- if settings.policy is defined %} {%- if index == 'so-logs-detections.alerts' %} -echo + echo echo "Setting up so-logs-detections.alerts-so policy..." curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://localhost:9200/_ilm/policy/{{ index }}-so" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }' echo -{%- else %} +{%- elif index == 'so-logs-soc' %} + echo + echo "Setting up so-soc-logs policy..." + curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://localhost:9200/_ilm/policy/so-soc-logs" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }' + echo + echo + echo "Setting up {{ index }}-logs policy..." + curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://localhost:9200/_ilm/policy/{{ index }}-logs" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }' + echo +{%- else %} + echo echo "Setting up {{ index }}-logs policy..." curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://localhost:9200/_ilm/policy/{{ index }}-logs" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }' echo From b952728b2c340e40870c2282fa1cd269a24a72c5 Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Mon, 26 Aug 2024 15:57:21 -0400 Subject: [PATCH 27/32] Fix policy load --- .../so-elastic-fleet-integration-policy-elastic-fleet-server | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-elastic-fleet-server b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-elastic-fleet-server index c304d5ba5..8f7c8b8b4 100644 --- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-elastic-fleet-server +++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-elastic-fleet-server @@ -17,7 +17,7 @@ for POLICYNAME in $POLICY; do printf "\nUpdating Policy: $POLICYNAME\n" # First get the Integration ID - elastic_fleet_integration_check "$POLICYNAME" "/opt/so/conf/elastic-fleet/integrations/fleet-server/fleet-server.json" + INTEGRATION_ID=$(/usr/sbin/so-elastic-fleet-agent-policy-view "$POLICYNAME" | jq -r '.item.package_policies[] | select(.package.name == "fleet_server") | .id') # Modify the default integration policy to update the policy_id and an with the correct naming UPDATED_INTEGRATION_POLICY=$(jq --arg policy_id "$POLICYNAME" --arg name "fleet_server-$POLICYNAME" ' From 726df310eebfa9e33c3bc6b75011d5557b1c67d0 Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Mon, 26 Aug 2024 16:15:56 -0400 Subject: [PATCH 28/32] Add context --- salt/manager/tools/sbin/soup | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index e634e0489..c74edf1eb 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -675,6 +675,7 @@ up_to_2.4.90() { so-yaml.py remove /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.password so-yaml.py add /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.config.password "$kafkatrimpass" so-yaml.py add /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.config.trustpass "$kafkatrust" + echo "If the Detection index exists, update the refresh_interval" so-elasticsearch-query so-detection*/_settings -X PUT -d '{"index":{"refresh_interval":"1s"}}' INSTALLEDVERSION=2.4.90 From 6043da4424150cfd500548961ef14e5768cdff73 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 27 Aug 2024 13:04:43 -0400 Subject: [PATCH 29/32] annotation updates --- salt/soc/soc_soc.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index ff7f8efd0..b4134baa7 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -192,6 +192,21 @@ soc: syntax: yaml helpLink: notifications.html forcedType: string + customAlerters: + description: "Specify custom notification alerters to use when the Sigma rule contains the following tag: so.alerters.customAlerters. This setting can be duplicated to create new custom alerter configurations. Specify one alerter name (Ex: 'email') per line. Alerters refers to ElastAlert 2 alerters, as documented at https://elastalert2.readthedocs.io. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key." + global: True + helpLink: notifications.html + forcedType: "[]string" + duplicates: True + multiline: True + customAlertersParams: + description: "Optional configuration parameters for custom notification alerters, used when the Sigma rule contains the following tag: so.params.customAlertersParams. This setting can be duplicated to create new custom alerter configurations. Use YAML format for these parameters, and reference the ElastAlert 2 documentation, located at https://elastalert2.readthedocs.io, for available alerters and their required configuration parameters. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key." + global: True + multiline: True + syntax: yaml + helpLink: notifications.html + duplicates: True + forcedType: string autoEnabledSigmaRules: default: &autoEnabledSigmaRules description: 'Sigma rules to automatically enable on initial import. Format is $Ruleset+$Level - for example, for the core community ruleset and critical level rules: core+critical. These will be applied based on role if defined and default if not.' From af80a78406df970eaf63ab24b64f23d4e2c3ae07 Mon Sep 17 00:00:00 2001 From: weslambert Date: Tue, 27 Aug 2024 13:08:35 -0400 Subject: [PATCH 30/32] Update pipeline version --- .../files/integrations/grid-nodes_general/import-evtx-logs.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json index 3b1a50560..fb8c31040 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json @@ -20,7 +20,7 @@ ], "data_stream.dataset": "import", "custom": "", - "processors": "- dissect:\n tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"\n- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n- drop_fields:\n fields: [\"host\"]\n ignore_missing: true\n- add_fields:\n target: data_stream\n fields:\n type: logs\n dataset: system.security\n- add_fields:\n target: event\n fields:\n dataset: system.security\n module: system\n imported: true\n- add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.security-1.43.0\n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.sysmon_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.sysmon_operational\n module: windows\n imported: true\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.sysmon_operational-1.38.0\n- if:\n equals:\n winlog.channel: 'Application'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.application\n - add_fields:\n target: event\n fields:\n dataset: system.application\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.application-1.43.0\n- if:\n equals:\n winlog.channel: 'System'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.system\n - add_fields:\n target: event\n fields:\n dataset: system.system\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.system-1.43.0\n \n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.powershell_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.powershell_operational\n module: windows\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.powershell_operational-1.38.0\n- add_fields:\n target: data_stream\n fields:\n dataset: import", + "processors": "- dissect:\n tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"\n- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n- drop_fields:\n fields: [\"host\"]\n ignore_missing: true\n- add_fields:\n target: data_stream\n fields:\n type: logs\n dataset: system.security\n- add_fields:\n target: event\n fields:\n dataset: system.security\n module: system\n imported: true\n- add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.security-1.59.0\n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.sysmon_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.sysmon_operational\n module: windows\n imported: true\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.sysmon_operational-1.45.1\n- if:\n equals:\n winlog.channel: 'Application'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.application\n - add_fields:\n target: event\n fields:\n dataset: system.application\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.application-1.59.0\n- if:\n equals:\n winlog.channel: 'System'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.system\n - add_fields:\n target: event\n fields:\n dataset: system.system\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.system-1.59.0\n \n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.powershell_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.powershell_operational\n module: windows\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.powershell_operational-1.45.1\n- add_fields:\n target: data_stream\n fields:\n dataset: import", "tags": [ "import" ] From f19a35ff06c43a0292782015e206910e6aff8b1d Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 28 Aug 2024 09:32:25 -0400 Subject: [PATCH 31/32] move custom alerters to subgroup; avoid false positives on log check --- salt/common/tools/sbin/so-log-check | 1 + salt/soc/soc_soc.yaml | 31 +++++++++++++++-------------- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check index 25ca4721f..240bde6a0 100755 --- a/salt/common/tools/sbin/so-log-check +++ b/salt/common/tools/sbin/so-log-check @@ -147,6 +147,7 @@ if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|status 200" # false positive (request successful, contained error string in content) EXCLUDED_ERRORS="$EXCLUDED_ERRORS|app_layer.error" # false positive (suricata 7) in stats.log e.g. app_layer.error.imap.parser | Total | 0 EXCLUDED_ERRORS="$EXCLUDED_ERRORS|is not an ip string literal" # false positive (Open Canary logging out blank IP addresses) + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|syncing rule" # false positive (rule sync log line includes rule name which can contain 'error') fi if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index b4134baa7..308044d50 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -192,21 +192,22 @@ soc: syntax: yaml helpLink: notifications.html forcedType: string - customAlerters: - description: "Specify custom notification alerters to use when the Sigma rule contains the following tag: so.alerters.customAlerters. This setting can be duplicated to create new custom alerter configurations. Specify one alerter name (Ex: 'email') per line. Alerters refers to ElastAlert 2 alerters, as documented at https://elastalert2.readthedocs.io. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key." - global: True - helpLink: notifications.html - forcedType: "[]string" - duplicates: True - multiline: True - customAlertersParams: - description: "Optional configuration parameters for custom notification alerters, used when the Sigma rule contains the following tag: so.params.customAlertersParams. This setting can be duplicated to create new custom alerter configurations. Use YAML format for these parameters, and reference the ElastAlert 2 documentation, located at https://elastalert2.readthedocs.io, for available alerters and their required configuration parameters. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key." - global: True - multiline: True - syntax: yaml - helpLink: notifications.html - duplicates: True - forcedType: string + additionalUserDefinedNotifications: + customAlerters: + description: "Specify custom notification alerters to use when the Sigma rule contains the following tag: so.alerters.customAlerters. This setting can be duplicated to create new custom alerter configurations. Specify one alerter name (Ex: 'email') per line. Alerters refers to ElastAlert 2 alerters, as documented at https://elastalert2.readthedocs.io. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key." + global: True + helpLink: notifications.html + forcedType: "[]string" + duplicates: True + multiline: True + customAlertersParams: + description: "Optional configuration parameters for custom notification alerters, used when the Sigma rule contains the following tag: so.params.customAlertersParams. This setting can be duplicated to create new custom alerter configurations. Use YAML format for these parameters, and reference the ElastAlert 2 documentation, located at https://elastalert2.readthedocs.io, for available alerters and their required configuration parameters. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key." + global: True + multiline: True + syntax: yaml + helpLink: notifications.html + duplicates: True + forcedType: string autoEnabledSigmaRules: default: &autoEnabledSigmaRules description: 'Sigma rules to automatically enable on initial import. Format is $Ruleset+$Level - for example, for the core community ruleset and critical level rules: core+critical. These will be applied based on role if defined and default if not.' From 89a1e2500ee41f723486e0f0d562e023d3dd7f68 Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Wed, 28 Aug 2024 16:50:11 -0400 Subject: [PATCH 32/32] Exclude logstash startup errors --- salt/common/tools/sbin/so-log-check | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check index 240bde6a0..300732587 100755 --- a/salt/common/tools/sbin/so-log-check +++ b/salt/common/tools/sbin/so-log-check @@ -95,6 +95,8 @@ if [[ $EXCLUDE_STARTUP_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|shutdown process" # server not yet ready (logstash waiting on elastic) EXCLUDED_ERRORS="$EXCLUDED_ERRORS|contain valid certificates" # server not yet ready (logstash waiting on elastic) EXCLUDED_ERRORS="$EXCLUDED_ERRORS|failedaction" # server not yet ready (logstash waiting on elastic) + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|block in start_workers" # server not yet ready (logstash waiting on elastic) + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|block in buffer_initialize" # server not yet ready (logstash waiting on elastic) EXCLUDED_ERRORS="$EXCLUDED_ERRORS|no route to host" # server not yet ready EXCLUDED_ERRORS="$EXCLUDED_ERRORS|not running" # server not yet ready EXCLUDED_ERRORS="$EXCLUDED_ERRORS|unavailable" # server not yet ready