diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check index 8f7e29d51..25ca4721f 100755 --- a/salt/common/tools/sbin/so-log-check +++ b/salt/common/tools/sbin/so-log-check @@ -206,7 +206,7 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|detect-parse" # Suricata encountering a malformed rule EXCLUDED_ERRORS="$EXCLUDED_ERRORS|integrity check failed" # Detections: Exclude false positive due to automated testing EXCLUDED_ERRORS="$EXCLUDED_ERRORS|syncErrors" # Detections: Not an actual error - EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Provided Grok expressions do not match field value\\: \\[unprovisioned\\]" # SOC log: before fields.status was changed to fields.licenseStatus + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Initialized license manager" # SOC log: before fields.status was changed to fields.licenseStatus fi RESULT=0 diff --git a/salt/common/tools/sbin_jinja/so-raid-status b/salt/common/tools/sbin_jinja/so-raid-status index 6cd8b84de..3fe238c23 100755 --- a/salt/common/tools/sbin_jinja/so-raid-status +++ b/salt/common/tools/sbin_jinja/so-raid-status @@ -9,6 +9,9 @@ . /usr/sbin/so-common +software_raid=("SOSMN" "SOSMN-DE02" "SOSSNNV" "SOSSNNV-DE02" "SOS10k-DE02" "SOS10KNV" "SOS10KNV-DE02" "SOS10KNV-DE02" "SOS2000-DE02" "SOS-GOFAST-LT-DE02" "SOS-GOFAST-MD-DE02" "SOS-GOFAST-HV-DE02") +hardware_raid=("SOS1000" "SOS1000F" "SOSSN7200" "SOS5000" "SOS4000") + {%- if salt['grains.get']('sosmodel', '') %} {%- set model = salt['grains.get']('sosmodel') %} model={{ model }} @@ -16,33 +19,42 @@ model={{ model }} if [[ $model =~ ^(SO2AMI01|SO2AZI01|SO2GCI01)$ ]]; then exit 0 fi + +for i in "${software_raid[@]}"; do + if [[ "$model" == $i ]]; then + is_softwareraid=true + is_hwraid=false + break + fi +done + +for i in "${hardware_raid[@]}"; do + if [[ "$model" == $i ]]; then + is_softwareraid=false + is_hwraid=true + break + fi +done + {%- else %} echo "This is not an appliance" exit 0 {%- endif %} -if [[ $model =~ ^(SOS10K|SOS500|SOS1000|SOS1000F|SOS4000|SOSSN7200|SOSSNNV|SOSMN)$ ]]; then - is_bossraid=true -fi -if [[ $model =~ ^(SOSSNNV|SOSMN)$ ]]; then - is_swraid=true -fi -if [[ $model =~ ^(SOS10K|SOS500|SOS1000|SOS1000F|SOS4000|SOSSN7200)$ ]]; then - is_hwraid=true -fi check_nsm_raid() { PERCCLI=$(/opt/raidtools/perccli/perccli64 /c0/v0 show|grep RAID|grep Optl) MEGACTL=$(/opt/raidtools/megasasctl |grep optimal) - - if [[ $APPLIANCE == '1' ]]; then + if [[ "$model" == "SOS500" || "$model" == "SOS500-DE02" ]]; then + #This doesn't have raid + HWRAID=0 + else if [[ -n $PERCCLI ]]; then HWRAID=0 elif [[ -n $MEGACTL ]]; then HWRAID=0 else HWRAID=1 - fi - + fi fi } @@ -50,17 +62,27 @@ check_nsm_raid() { check_boss_raid() { MVCLI=$(/usr/local/bin/mvcli info -o vd |grep status |grep functional) MVTEST=$(/usr/local/bin/mvcli info -o vd | grep "No adapter") + BOSSNVMECLI=$(/usr/local/bin/mnv_cli info -o vd -i 0 | grep Functional) - # Check to see if this is a SM based system - if [[ -z $MVTEST ]]; then - if [[ -n $MVCLI ]]; then + # Is this NVMe Boss Raid? + if [[ "$model" =~ "-DE02" ]]; then + if [[ -n $BOSSNVMECLI ]]; then BOSSRAID=0 else BOSSRAID=1 fi else - # This doesn't have boss raid so lets make it 0 - BOSSRAID=0 + # Check to see if this is a SM based system + if [[ -z $MVTEST ]]; then + if [[ -n $MVCLI ]]; then + BOSSRAID=0 + else + BOSSRAID=1 + fi + else + # This doesn't have boss raid so lets make it 0 + BOSSRAID=0 + fi fi } @@ -79,14 +101,13 @@ SWRAID=0 BOSSRAID=0 HWRAID=0 -if [[ $is_hwraid ]]; then +if [[ "$is_hwraid" == "true" ]]; then check_nsm_raid + check_boss_raid fi -if [[ $is_bossraid ]]; then - check_boss_raid -fi -if [[ $is_swraid ]]; then +if [[ "$is_softwareraid" == "true" ]]; then check_software_raid + check_boss_raid fi sum=$(($SWRAID + $BOSSRAID + $HWRAID)) diff --git a/salt/elastalert/soc_elastalert.yaml b/salt/elastalert/soc_elastalert.yaml index 435c5be6a..905fd3884 100644 --- a/salt/elastalert/soc_elastalert.yaml +++ b/salt/elastalert/soc_elastalert.yaml @@ -3,8 +3,8 @@ elastalert: description: You can enable or disable Elastalert. helpLink: elastalert.html alerter_parameters: - title: Alerter Parameters - description: Optional configuration parameters for additional alerters that can be enabled for all Sigma rules. Filter for 'Alerter' in this Configuration screen to find the setting that allows these alerters to be enabled within the SOC ElastAlert module. Use YAML format for these parameters, and reference the ElastAlert 2 documentation, located at https://elastalert2.readthedocs.io, for available alerters and their required configuration parameters. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key. + title: Custom Configuration Parameters + description: Optional configuration parameters made available as defaults for all rules and alerters. Use YAML format for these parameters, and reference the ElastAlert 2 documentation, located at https://elastalert2.readthedocs.io, for available configuration parameters. Requires a valid Security Onion license key. global: True multiline: True syntax: yaml diff --git a/salt/elasticfleet/defaults.yaml b/salt/elasticfleet/defaults.yaml index 2d9ab97a1..48b24809e 100644 --- a/salt/elasticfleet/defaults.yaml +++ b/salt/elasticfleet/defaults.yaml @@ -97,6 +97,7 @@ elasticfleet: - symantec_endpoint - system - tcp + - tenable_io - tenable_sc - ti_abusech - ti_anomali diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index b18ab5a67..7201df25e 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -9282,6 +9282,190 @@ elasticsearch: set_priority: priority: 50 min_age: 30d + so-logs-tenable_io_x_asset: + index_sorting: False + index_template: + index_patterns: + - "logs-tenable_io.asset-*" + template: + settings: + index: + lifecycle: + name: so-logs-tenable_io.asset-logs + number_of_replicas: 0 + composed_of: + - "logs-tenable_io.asset@package" + - "logs-tenable_io.asset@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + ignore_missing_component_templates: + - logs-tenable_io.asset@custom + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-tenable_io_x_plugin: + index_sorting: False + index_template: + index_patterns: + - "logs-tenable_io.plugin-*" + template: + settings: + index: + lifecycle: + name: so-logs-tenable_io.plugin-logs + number_of_replicas: 0 + composed_of: + - "logs-tenable_io.plugin@package" + - "logs-tenable_io.plugin@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + ignore_missing_component_templates: + - logs-tenable_io.plugin@custom + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-tenable_io_x_scan: + index_sorting: False + index_template: + index_patterns: + - "logs-tenable_io.scan-*" + template: + settings: + index: + lifecycle: + name: so-logs-tenable_io.scan-logs + number_of_replicas: 0 + composed_of: + - "logs-tenable_io.scan@package" + - "logs-tenable_io.scan@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + ignore_missing_component_templates: + - logs-tenable_io.scan@custom + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-tenable_io_x_vulnerability: + index_sorting: False + index_template: + index_patterns: + - "logs-tenable_io.vulnerability-*" + template: + settings: + index: + lifecycle: + name: so-logs-tenable_io.vulnerability-logs + number_of_replicas: 0 + composed_of: + - "logs-tenable_io.vulnerability@package" + - "logs-tenable_io.vulnerability@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + ignore_missing_component_templates: + - logs-tenable_io.vulnerability@custom + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d so-logs-tenable_sc_x_asset: index_sorting: false index_template: diff --git a/salt/elasticsearch/files/ingest-dynamic/common b/salt/elasticsearch/files/ingest-dynamic/common index 836b8d4af..93e7f1cca 100644 --- a/salt/elasticsearch/files/ingest-dynamic/common +++ b/salt/elasticsearch/files/ingest-dynamic/common @@ -62,6 +62,7 @@ { "split": { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "separator": "\\.", "target_field": "dataset_tag_temp" } }, { "append": { "if": "ctx.dataset_tag_temp != null", "field": "tags", "value": "{{dataset_tag_temp.1}}" } }, { "grok": { "if": "ctx.http?.response?.status_code != null", "field": "http.response.status_code", "patterns": ["%{NUMBER:http.response.status_code:long} %{GREEDYDATA}"]} }, + { "set": { "if": "ctx?.metadata?.kafka != null" , "field": "kafka.id", "value": "{{metadata.kafka.partition}}{{metadata.kafka.offset}}{{metadata.kafka.timestamp}}", "ignore_failure": true } }, { "remove": { "field": [ "message2", "type", "fields", "category", "module", "dataset", "dataset_tag_temp", "event.dataset_temp" ], "ignore_missing": true, "ignore_failure": true } } {%- endraw %} {%- if HIGHLANDER %} diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index 085aab7f0..d30706837 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -466,6 +466,13 @@ elasticsearch: so-logs-sonicwall_firewall_x_log: *indexSettings so-logs-snort_x_log: *indexSettings so-logs-symantec_endpoint_x_log: *indexSettings + so-logs-tenable_io_x_asset: *indexSettings + so-logs-tenable_io_x_plugin: *indexSettings + so-logs-tenable_io_x_scan: *indexSettings + so-logs-tenable_io_x_vulnerability: *indexSettings + so-logs-tenable_sc_x_asset: *indexSettings + so-logs-tenable_sc_x_plugin: *indexSettings + so-logs-tenable_sc_x_vulnerability: *indexSettings so-logs-ti_abusech_x_malware: *indexSettings so-logs-ti_abusech_x_malwarebazaar: *indexSettings so-logs-ti_abusech_x_threatfox: *indexSettings diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load index b00fcbedf..7d3894950 100755 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load @@ -10,10 +10,16 @@ {%- for index, settings in ES_INDEX_SETTINGS.items() %} {%- if settings.policy is defined %} +{%- if index == 'so-logs-detections.alerts' %} echo -echo "Setting up {{ index }}-logs policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://localhost:9200/_ilm/policy/{{ index }}-logs" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }' -echo + echo "Setting up so-logs-detections.alerts-so policy..." + curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://localhost:9200/_ilm/policy/{{ index }}-so" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }' + echo +{%- else %} + echo "Setting up {{ index }}-logs policy..." + curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://localhost:9200/_ilm/policy/{{ index }}-logs" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }' + echo +{%- endif %} {%- endif %} {%- endfor %} echo diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load index 5d1e88798..76b1cc193 100755 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load @@ -5,7 +5,6 @@ # Elastic License 2.0. {%- import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %} {% from 'vars/globals.map.jinja' import GLOBALS %} -{%- set SUPPORTED_PACKAGES = salt['pillar.get']('elasticfleet:packages', default=ELASTICFLEETDEFAULTS.elasticfleet.packages, merge=True) %} STATE_FILE_INITIAL=/opt/so/state/estemplates_initial_load_attempt.txt STATE_FILE_SUCCESS=/opt/so/state/estemplates.txt @@ -68,9 +67,9 @@ if [ ! -f $STATE_FILE_SUCCESS ]; then echo -n "Waiting for ElasticSearch..." retry 240 1 "so-elasticsearch-query / -k --output /dev/null --silent --head --fail" || fail "Connection attempt timed out. Unable to connect to ElasticSearch. \nPlease try: \n -checking log(s) in /var/log/elasticsearch/\n -running 'sudo docker ps' \n -running 'sudo so-elastic-restart'" {% if GLOBALS.role != 'so-heavynode' %} - SESSIONCOOKIE=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}') - INSTALLED=$(elastic_fleet_package_is_installed endpoint }} ) - if [ "$INSTALLED" != "installed" ]; then + TEMPLATE="logs-endpoint.alerts@package" + INSTALLED=$(so-elasticsearch-query _component_template/$TEMPLATE | jq -r .component_templates[0].name) + if [ "$INSTALLED" != "$TEMPLATE" ]; then echo echo "Packages not yet installed." echo diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index d2f63e4ad..ff7f8efd0 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -103,12 +103,95 @@ soc: description: Show AI summaries for ElastAlert rules. global: True additionalAlerters: - title: Additional Alerters - description: Specify additional alerters to enable for all Sigma rules, one alerter name per line. Alerters refers to ElastAlert 2 alerters, as documented at https://elastalert2.readthedocs.io. Note that the configuration parameters for these alerters must be provided in the ElastAlert configuration section. Filter for 'Alerter' to find this related setting. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key. + title: "Notifications: Sev 0/Default Alerters" + description: "Specify default alerters to enable for outbound notifications. These alerters will be used unless overridden by higher severity alerter settings. Specify one alerter name (Ex: 'email') per line. Alerters refers to ElastAlert 2 alerters, as documented at https://elastalert2.readthedocs.io. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key." global: True - helpLink: sigma.html + helpLink: notifications.html forcedType: "[]string" multiline: True + additionalSev0AlertersParams: + title: "Notifications: Sev 0/Default Parameters" + description: Optional configuration parameters for default alerters. Use YAML format for these parameters, and reference the ElastAlert 2 documentation, located at https://elastalert2.readthedocs.io, for available alerters and their required configuration parameters. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key. + global: True + multiline: True + syntax: yaml + helpLink: notifications.html + forcedType: string + additionalSev1Alerters: + title: "Notifications: Sev 1/Informational Alerters" + description: "Specify specific alerters to use when alerting at the info severity level or higher. These alerters will be used unless overridden by higher severity alerter settings. Specify one alerter name (Ex: 'email') per line. Alerters refers to ElastAlert 2 alerters, as documented at https://elastalert2.readthedocs.io. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key." + global: True + helpLink: notifications.html + forcedType: "[]string" + multiline: True + additionalSev1AlertersParams: + title: "Notifications: Sev 1/Informational Parameters" + description: Optional configuration parameters for informational severity alerters. Info level is less severe than 'Low Severity'. Use YAML format for these parameters, and reference the ElastAlert 2 documentation, located at https://elastalert2.readthedocs.io, for available alerters and their required configuration parameters. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key. + global: True + multiline: True + syntax: yaml + helpLink: notifications.html + forcedType: string + additionalSev2Alerters: + title: "Notifications: Sev 2/Low Alerters" + description: "Specify specific alerters to use when alerting at the low severity level or higher. These alerters will be used unless overridden by higher severity alerter settings. Specify one alerter name (Ex: 'email') per line. Alerters refers to ElastAlert 2 alerters, as documented at https://elastalert2.readthedocs.io. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key." + global: True + helpLink: notifications.html + forcedType: "[]string" + multiline: True + additionalSev2AlertersParams: + title: "Notifications: Sev 2/Low Parameters" + description: Optional configuration parameters for low severity alerters. Use YAML format for these parameters, and reference the ElastAlert 2 documentation, located at https://elastalert2.readthedocs.io, for available alerters and their required configuration parameters. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key. + global: True + multiline: True + syntax: yaml + helpLink: notifications.html + forcedType: string + additionalSev3Alerters: + title: "Notifications: Sev 3/Medium Alerters" + description: "Specify specific alerters to use when alerting at the medium severity level or higher. These alerters will be used unless overridden by higher severity alerter settings. Specify one alerter name (Ex: 'email') per line. Alerters refers to ElastAlert 2 alerters, as documented at https://elastalert2.readthedocs.io. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key." + global: True + helpLink: notifications.html + forcedType: "[]string" + multiline: True + additionalSev3AlertersParams: + title: "Notifications: Sev 3/Medium Parameters" + description: Optional configuration parameters for medium severity alerters. Use YAML format for these parameters, and reference the ElastAlert 2 documentation, located at https://elastalert2.readthedocs.io, for available alerters and their required configuration parameters. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key. + global: True + multiline: True + syntax: yaml + helpLink: notifications.html + forcedType: string + additionalSev4Alerters: + title: "Notifications: Sev 4/High Alerters" + description: "Specify specific alerters to use when alerting at the high severity level or critical severity level. These alerters will be used unless overridden by critical severity alerter settings. Specify one alerter name (Ex: 'email') per line. Alerters refers to ElastAlert 2 alerters, as documented at https://elastalert2.readthedocs.io. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key." + global: True + helpLink: notifications.html + forcedType: "[]string" + multiline: True + additionalSev4AlertersParams: + title: "Notifications: Sev 4/High Parameters" + description: Optional configuration parameters for high severity alerters. Use YAML format for these parameters, and reference the ElastAlert 2 documentation, located at https://elastalert2.readthedocs.io, for available alerters and their required configuration parameters. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key. + global: True + multiline: True + syntax: yaml + helpLink: notifications.html + forcedType: string + additionalSev5Alerters: + title: "Notifications: Sev 5/Critical Alerters" + description: "Specify specific alerters to use when alerting at the critical severity level. Specify one alerter name (Ex: 'email') per line. Alerters refers to ElastAlert 2 alerters, as documented at https://elastalert2.readthedocs.io. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key." + global: True + helpLink: notifications.html + forcedType: "[]string" + multiline: True + additionalSev5AlertersParams: + title: "Notifications: Sev 5/Critical Parameters" + description: Optional configuration parameters for critical severity alerters. Use YAML format for these parameters, and reference the ElastAlert 2 documentation, located at https://elastalert2.readthedocs.io, for available alerters and their required configuration parameters. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key. + global: True + multiline: True + syntax: yaml + helpLink: notifications.html + forcedType: string autoEnabledSigmaRules: default: &autoEnabledSigmaRules description: 'Sigma rules to automatically enable on initial import. Format is $Ruleset+$Level - for example, for the core community ruleset and critical level rules: core+critical. These will be applied based on role if defined and default if not.'