From 000d15a53c60bf923b826602fcb93f3de7ec722c Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Fri, 29 Mar 2024 13:56:01 -0400 Subject: [PATCH 01/63] Kismet integration: TODO Elasticsearch mappings Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/elasticfleet/defaults.yaml | 5 + .../files/integrations-optional/kismet.json | 36 ++++ salt/elasticfleet/soc_elasticfleet.yaml | 26 +++ salt/elasticsearch/defaults.yaml | 43 +++++ salt/elasticsearch/files/ingest/kismet.ad_hoc | 10 ++ salt/elasticsearch/files/ingest/kismet.ap | 50 ++++++ .../elasticsearch/files/ingest/kismet.bridged | 16 ++ salt/elasticsearch/files/ingest/kismet.client | 29 ++++ salt/elasticsearch/files/ingest/kismet.common | 158 ++++++++++++++++++ salt/elasticsearch/files/ingest/kismet.device | 9 + salt/elasticsearch/files/ingest/kismet.seenby | 52 ++++++ salt/elasticsearch/files/ingest/kismet.wds | 10 ++ salt/elasticsearch/files/ingest/kismet.wds_ap | 22 +++ salt/elasticsearch/soc_elasticsearch.yaml | 1 + .../templates/component/ecs/kismet.json | 16 ++ 15 files changed, 483 insertions(+) create mode 100644 salt/elasticfleet/files/integrations-optional/kismet.json create mode 100644 salt/elasticsearch/files/ingest/kismet.ad_hoc create mode 100644 salt/elasticsearch/files/ingest/kismet.ap create mode 100644 salt/elasticsearch/files/ingest/kismet.bridged create mode 100644 salt/elasticsearch/files/ingest/kismet.client create mode 100644 salt/elasticsearch/files/ingest/kismet.common create mode 100644 salt/elasticsearch/files/ingest/kismet.device create mode 100644 salt/elasticsearch/files/ingest/kismet.seenby create mode 100644 salt/elasticsearch/files/ingest/kismet.wds create mode 100644 salt/elasticsearch/files/ingest/kismet.wds_ap create mode 100644 salt/elasticsearch/templates/component/ecs/kismet.json diff --git a/salt/elasticfleet/defaults.yaml b/salt/elasticfleet/defaults.yaml index 7b2d9d6a3..2af7e7532 100644 --- a/salt/elasticfleet/defaults.yaml +++ b/salt/elasticfleet/defaults.yaml @@ -118,3 +118,8 @@ elasticfleet: base_url: https://api.platform.sublimesecurity.com poll_interval: 5m limit: 100 + kismet: + base_url: http://localhost:2501 + poll_interval: 1m + api_key: + enabled_nodes: [] \ No newline at end of file diff --git a/salt/elasticfleet/files/integrations-optional/kismet.json b/salt/elasticfleet/files/integrations-optional/kismet.json new file mode 100644 index 000000000..9a333f31c --- /dev/null +++ b/salt/elasticfleet/files/integrations-optional/kismet.json @@ -0,0 +1,36 @@ +{% from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %} +{% raw %} +{ + "package": { + "name": "httpjson", + "version": "" + }, + "name": "kismet-logs", + "namespace": "so", + "description": "Kismet Logs", + "policy_id": "FleetServer_{% endraw %}{{ NAME }}{% raw %}", + "inputs": { + "generic-httpjson": { + "enabled": true, + "streams": { + "httpjson.generic": { + "enabled": true, + "vars": { + "data_stream.dataset": "kismet", + "request_url": "{% endraw %}{{ ELASTICFLEETMERGED.optional_integrations.kismet.base_url }}{% raw %}/devices/last-time/-600/devices.tjson", + "request_interval": "{% endraw %}{{ ELASTICFLEETMERGED.optional_integrations.kismet.poll_interval }}{% raw %}", + "request_method": "GET", + "request_transforms": "- set:\r\n target: header.Cookie\r\n value: 'KISMET={% endraw %}{{ ELASTICFLEETMERGED.optional_integrations.kismet.api_key }}{% raw %}'", + "request_redirect_headers_ban_list": [], + "oauth_scopes": [], + "processors": "", + "tags": [], + "pipeline": "kismet.common" + } + } + } + } + }, + "force": true +} +{% endraw %} \ No newline at end of file diff --git a/salt/elasticfleet/soc_elasticfleet.yaml b/salt/elasticfleet/soc_elasticfleet.yaml index 9a0cd0a91..206febcd7 100644 --- a/salt/elasticfleet/soc_elasticfleet.yaml +++ b/salt/elasticfleet/soc_elasticfleet.yaml @@ -79,3 +79,29 @@ elasticfleet: helpLink: elastic-fleet.html advanced: True forcedType: int + kismet: + base_url: + description: Base URL for Kismet. + global: True + helpLink: elastic-fleet.html + advanced: True + forcedType: string + poll_interval: + description: Poll interval for wireless device data from Kismet. Integration is currently configured to report devices seen as active by any Kismet sensor within the last 600 seconds of polling. + global: True + helpLink: elastic-fleet.html + advanced: True + forcedType: string + api_key: + description: API key for Kismet. + global: True + helpLink: elastic-fleet.html + advanced: True + forcedType: string + sensitive: True + enabled_nodes: + description: Fleet nodes with the Kismet integration enabled. Enter one per line. + global: True + helpLink: elastic-fleet.html + advanced: True + forcedType: "[]string" diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index c70b0419a..048dd0c7f 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -10491,6 +10491,49 @@ elasticsearch: set_priority: priority: 50 min_age: 30d + so-kismet: + index_sorting: false + index_template: + composed_of: + - kismet-mappings + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-kismet-so* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-kismet-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d so-logstash: index_sorting: false index_template: diff --git a/salt/elasticsearch/files/ingest/kismet.ad_hoc b/salt/elasticsearch/files/ingest/kismet.ad_hoc new file mode 100644 index 000000000..8cbc9cd2b --- /dev/null +++ b/salt/elasticsearch/files/ingest/kismet.ad_hoc @@ -0,0 +1,10 @@ +{ + "processors": [ + { + "rename": { + "field": "message2.kismet_device_base_macaddr", + "target_field": "wireless.bssid" + } + } + ] +} \ No newline at end of file diff --git a/salt/elasticsearch/files/ingest/kismet.ap b/salt/elasticsearch/files/ingest/kismet.ap new file mode 100644 index 000000000..1b8cbb80e --- /dev/null +++ b/salt/elasticsearch/files/ingest/kismet.ap @@ -0,0 +1,50 @@ +{ + "processors": [ + { + "rename": { + "field": "message2.dot11_device.dot11_device_last_beaconed_ssid_record.dot11_advertisedssid_cloaked", + "target_field": "wireless.ssid_cloaked", + "if": "ctx?.message2?.dot11_device?.dot11_device_last_beaconed_ssid_record?.dot11_advertisedssid_cloaked != null" + } + }, + { + "rename": { + "field": "message2.dot11_device.dot11_device_last_beaconed_ssid_record.dot11_advertisedssid_ssid", + "target_field": "wireless.ssid", + "if": "ctx?.message2?.dot11_device?.dot11_device_last_beaconed_ssid_record?.dot11_advertisedssid_ssid != null" + } + }, + { + "set": { + "field": "wireless.ssid", + "value": "Hidden", + "if": "ctx?.wireless?.ssid_cloaked != null && ctx?.wireless?.ssid_cloaked == 1" + } + }, + { + "rename": { + "field": "message2.dot11_device.dot11_device_last_beaconed_ssid_record.dot11_advertisedssid_dot11e_channel_utilization_perc", + "target_field": "wireless.channel_utilization", + "if": "ctx?.message2?.dot11_device?.dot11_device_last_beaconed_ssid_record?.dot11_advertisedssid_dot11e_channel_utilization_perc != null" + } + }, + { + "rename": { + "field": "message2.dot11_device.dot11_device_last_bssid", + "target_field": "wireless.bssid" + } + }, + { + "foreach": { + "field": "message2.dot11_device.dot11_device_associated_client_map", + "processor": { + "append": { + "field": "wireless.associated_clients", + "value": "{{_ingest._key}}" + } + }, + "if": "ctx?.message2?.dot11_device?.dot11_device_associated_client_map != null" + } + } + ] +} \ No newline at end of file diff --git a/salt/elasticsearch/files/ingest/kismet.bridged b/salt/elasticsearch/files/ingest/kismet.bridged new file mode 100644 index 000000000..5eee3b78c --- /dev/null +++ b/salt/elasticsearch/files/ingest/kismet.bridged @@ -0,0 +1,16 @@ +{ + "processors": [ + { + "rename": { + "field": "message2.kismet_device_base_macaddr", + "target_field": "client.mac" + } + }, + { + "rename": { + "field": "message2.dot11_device.dot11_device_last_bssid", + "target_field": "wireless.bssid" + } + } + ] +} \ No newline at end of file diff --git a/salt/elasticsearch/files/ingest/kismet.client b/salt/elasticsearch/files/ingest/kismet.client new file mode 100644 index 000000000..8b3d3069b --- /dev/null +++ b/salt/elasticsearch/files/ingest/kismet.client @@ -0,0 +1,29 @@ +{ + "processors": [ + { + "rename": { + "field": "message2.kismet_device_base_macaddr", + "target_field": "client.mac" + } + }, + { + "rename": { + "field": "message2.dot11_device.dot11_device_last_bssid", + "target_field": "wireless.last_connected_bssid", + "if": "ctx?.message2?.dot11_device?.dot11_device_last_bssid != null" + } + }, + { + "foreach": { + "field": "message2.dot11_device.dot11_device_client_map", + "processor": { + "append": { + "field": "wireless.known_connected_bssid", + "value": "{{_ingest._key}}" + } + }, + "if": "ctx?.message2?.dot11_device?.dot11_device_client_map != null" + } + } + ] +} \ No newline at end of file diff --git a/salt/elasticsearch/files/ingest/kismet.common b/salt/elasticsearch/files/ingest/kismet.common new file mode 100644 index 000000000..95eb29b73 --- /dev/null +++ b/salt/elasticsearch/files/ingest/kismet.common @@ -0,0 +1,158 @@ +{ + "processors": [ + { + "json": { + "field": "message", + "target_field": "message2" + } + }, + { + "date": { + "field": "message2.kismet_device_base_mod_time", + "formats": [ + "epoch_second" + ], + "target_field": "@timestamp" + } + }, + { + "set": { + "field": "event.category", + "value": "network" + } + }, + { + "dissect": { + "field": "message2.kismet_device_base_type", + "pattern": "%{wifi} %{device_type}" + } + }, + { + "lowercase": { + "field": "device_type" + } + }, + { + "set": { + "field": "event.dataset", + "value": "kismet.{{device_type}}" + } + }, + { + "set": { + "field": "event.dataset", + "value": "kismet.wds_ap", + "if": "ctx?.device_type == 'wds ap'" + } + }, + { + "set": { + "field": "event.dataset", + "value": "kismet.ad_hoc", + "if": "ctx?.device_type == 'ad-hoc'" + } + }, + { + "set": { + "field": "event.module", + "value": "kismet" + } + }, + { + "rename": { + "field": "message2.kismet_device_base_packets_tx_total", + "target_field": "source.packets" + } + }, + { + "rename": { + "field": "message2.kismet_device_base_num_alerts", + "target_field": "kismet.alerts.count" + } + }, + { + "rename": { + "field": "message2.kismet_device_base_channel", + "target_field": "wireless.channel", + "if": "ctx?.message2?.kismet_device_base_channel != ''" + } + }, + { + "rename": { + "field": "message2.kismet_device_base_frequency", + "target_field": "wireless.frequency", + "if": "ctx?.message2?.kismet_device_base_frequency != 0" + } + }, + { + "rename": { + "field": "message2.kismet_device_base_last_time", + "target_field": "kismet.last_seen" + } + }, + { + "date": { + "field": "kismet.last_seen", + "formats": [ + "epoch_second" + ], + "target_field": "kismet.last_seen" + } + }, + { + "rename": { + "field": "message2.kismet_device_base_first_time", + "target_field": "kismet.first_seen" + } + }, + { + "date": { + "field": "kismet.first_seen", + "formats": [ + "epoch_second" + ], + "target_field": "kismet.first_seen" + } + }, + { + "rename": { + "field": "message2.kismet_device_base_seenby", + "target_field": "kismet.seenby" + } + }, + { + "foreach": { + "field": "kismet.seenby", + "processor": { + "pipeline": { + "name": "kismet.seenby" + } + } + } + }, + { + "rename": { + "field": "message2.kismet_device_base_manuf", + "target_field": "device.manufacturer" + } + }, + { + "pipeline": { + "name": "{{event.dataset}}" + } + }, + { + "remove": { + "field": [ + "message2", + "message", + "device_type", + "wifi", + "agent", + "host" + ], + "ignore_failure": true + } + } + ] +} \ No newline at end of file diff --git a/salt/elasticsearch/files/ingest/kismet.device b/salt/elasticsearch/files/ingest/kismet.device new file mode 100644 index 000000000..49d0c7ad7 --- /dev/null +++ b/salt/elasticsearch/files/ingest/kismet.device @@ -0,0 +1,9 @@ +{ + "processors": [ + { + "pipeline": { + "name": "kismet.client" + } + } + ] +} \ No newline at end of file diff --git a/salt/elasticsearch/files/ingest/kismet.seenby b/salt/elasticsearch/files/ingest/kismet.seenby new file mode 100644 index 000000000..d41220d76 --- /dev/null +++ b/salt/elasticsearch/files/ingest/kismet.seenby @@ -0,0 +1,52 @@ +{ + "processors": [ + { + "rename": { + "field": "_ingest._value.kismet_common_seenby_num_packets", + "target_field": "_ingest._value.packets_seen", + "ignore_missing": true + } + }, + { + "rename": { + "field": "_ingest._value.kismet_common_seenby_uuid", + "target_field": "_ingest._value.serial_number", + "ignore_missing": true + } + }, + { + "rename": { + "field": "_ingest._value.kismet_common_seenby_first_time", + "target_field": "_ingest._value.first_seen", + "ignore_missing": true + } + }, + { + "rename": { + "field": "_ingest._value.kismet_common_seenby_last_time", + "target_field": "_ingest._value.last_seen", + "ignore_missing": true + } + }, + { + "date": { + "field": "_ingest._value.first_seen", + "formats": [ + "epoch_second" + ], + "target_field": "_ingest._value.first_seen", + "ignore_failure": true + } + }, + { + "date": { + "field": "_ingest._value.last_seen", + "formats": [ + "epoch_second" + ], + "target_field": "_ingest._value.last_seen", + "ignore_failure": true + } + } + ] +} \ No newline at end of file diff --git a/salt/elasticsearch/files/ingest/kismet.wds b/salt/elasticsearch/files/ingest/kismet.wds new file mode 100644 index 000000000..1e426c463 --- /dev/null +++ b/salt/elasticsearch/files/ingest/kismet.wds @@ -0,0 +1,10 @@ +{ + "processors": [ + { + "rename": { + "field": "message2.kismet_device_base_macaddr", + "target_field": "client.mac" + } + } + ] +} \ No newline at end of file diff --git a/salt/elasticsearch/files/ingest/kismet.wds_ap b/salt/elasticsearch/files/ingest/kismet.wds_ap new file mode 100644 index 000000000..7f43d43fd --- /dev/null +++ b/salt/elasticsearch/files/ingest/kismet.wds_ap @@ -0,0 +1,22 @@ +{ + "processors": [ + { + "rename": { + "field": "message2.kismet_device_base_commonname", + "target_field": "wireless.bssid" + } + }, + { + "foreach": { + "field": "message2.dot11_device.dot11_device_associated_client_map", + "processor": { + "append": { + "field": "wireless.associated_clients", + "value": "{{_ingest._key}}" + } + }, + "if": "ctx?.message2?.dot11_device?.dot11_device_associated_client_map != null" + } + } + ] +} \ No newline at end of file diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index e68d0441b..c684c6154 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -511,6 +511,7 @@ elasticsearch: so-suricata: *indexSettings so-import: *indexSettings so-kratos: *indexSettings + so-kismet: *indexSettings so-logstash: *indexSettings so-redis: *indexSettings so-strelka: *indexSettings diff --git a/salt/elasticsearch/templates/component/ecs/kismet.json b/salt/elasticsearch/templates/component/ecs/kismet.json new file mode 100644 index 000000000..d388b7127 --- /dev/null +++ b/salt/elasticsearch/templates/component/ecs/kismet.json @@ -0,0 +1,16 @@ +{ + "_meta": { + "documentation": "https://www.elastic.co/guide/en/ecs/current/ecs-base.html", + "ecs_version": "1.12.2" + }, + "template": { + "mappings": { + "properties": { + "kismet_mapping_placeholder": { + "type": "keyword", + "ignore_above": 1024 + } + } + } + } +} \ No newline at end of file From 2206553e038a8c6fb32b0c680b96b79f1137c583 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 10 Apr 2024 09:49:21 -0400 Subject: [PATCH 02/63] Update analyst.json --- salt/elasticsearch/roles/analyst.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/elasticsearch/roles/analyst.json b/salt/elasticsearch/roles/analyst.json index 2fd10ab47..90ff95ad4 100644 --- a/salt/elasticsearch/roles/analyst.json +++ b/salt/elasticsearch/roles/analyst.json @@ -27,7 +27,8 @@ "monitor", "read", "read_cross_cluster", - "view_index_metadata" + "view_index_metadata", + "write" ] } ], From 4097e1d81ab07f4f9e9b3e55599b9b66b63ae7a1 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Wed, 10 Apr 2024 16:10:27 -0400 Subject: [PATCH 03/63] Create mappings for Kismet integration Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/elasticsearch/defaults.yaml | 4 ++ .../templates/component/ecs/device.json | 36 ++++++++++++++++++ .../templates/component/ecs/kismet.json | 22 +++++++++-- .../templates/component/ecs/network.json | 37 +++++++++++++++++++ 4 files changed, 96 insertions(+), 3 deletions(-) create mode 100644 salt/elasticsearch/templates/component/ecs/device.json diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 048dd0c7f..db1255dad 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -10496,6 +10496,10 @@ elasticsearch: index_template: composed_of: - kismet-mappings + - source-mappings + - client-mappings + - device-mappings + - network-mappings - so-fleet_globals-1 - so-fleet_agent_id_verification-1 data_stream: diff --git a/salt/elasticsearch/templates/component/ecs/device.json b/salt/elasticsearch/templates/component/ecs/device.json new file mode 100644 index 000000000..a281f2c1e --- /dev/null +++ b/salt/elasticsearch/templates/component/ecs/device.json @@ -0,0 +1,36 @@ +{ + "_meta": { + "documentation": "https://www.elastic.co/guide/en/ecs/current/ecs-device.html", + "ecs_version": "1.12.2" + }, + "template": { + "mappings": { + "properties": { + "device": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "manufacturer": { + "ignore_above": 1024, + "type": "keyword" + }, + "model": { + "properties": { + "identifier": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + } + } + } + } +} \ No newline at end of file diff --git a/salt/elasticsearch/templates/component/ecs/kismet.json b/salt/elasticsearch/templates/component/ecs/kismet.json index d388b7127..a03236ab8 100644 --- a/salt/elasticsearch/templates/component/ecs/kismet.json +++ b/salt/elasticsearch/templates/component/ecs/kismet.json @@ -6,9 +6,25 @@ "template": { "mappings": { "properties": { - "kismet_mapping_placeholder": { - "type": "keyword", - "ignore_above": 1024 + "kismet": { + "properties": { + "alerts": { + "properties": { + "count": { + "type": "long" + } + } + }, + "first_seen": { + "type": "date" + }, + "last_seen": { + "type": "date" + }, + "seenby": { + "type": "nested" + } + } } } } diff --git a/salt/elasticsearch/templates/component/ecs/network.json b/salt/elasticsearch/templates/component/ecs/network.json index c2e35efd0..cc0f9d288 100644 --- a/salt/elasticsearch/templates/component/ecs/network.json +++ b/salt/elasticsearch/templates/component/ecs/network.json @@ -77,6 +77,43 @@ "type": "keyword" } } + }, + "wireless": { + "properties": { + "associated_clients": { + "ignore_above": 1024, + "type": "keyword" + }, + "bssid": { + "ignore_above": 1024, + "type": "keyword" + }, + "channel": { + "ignore_above": 1024, + "type": "keyword" + }, + "channel_utilization": { + "type": "float" + }, + "frequency": { + "type": "double" + }, + "ssid": { + "ignore_above": 1024, + "type": "keyword" + }, + "ssid_cloaked": { + "type": "boolean" + }, + "known_connected_bssid": { + "ignore_above": 1024, + "type": "keyword" + }, + "last_connected_bssid": { + "ignore_above": 1024, + "type": "keyword" + } + } } } } From 2ab9cbba6131c508ee27293e759b6e62753c323d Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Wed, 10 Apr 2024 16:12:22 -0400 Subject: [PATCH 04/63] Update wording for Kismet poll interval annotation Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/elasticfleet/soc_elasticfleet.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticfleet/soc_elasticfleet.yaml b/salt/elasticfleet/soc_elasticfleet.yaml index 206febcd7..7ed97e6ec 100644 --- a/salt/elasticfleet/soc_elasticfleet.yaml +++ b/salt/elasticfleet/soc_elasticfleet.yaml @@ -87,7 +87,7 @@ elasticfleet: advanced: True forcedType: string poll_interval: - description: Poll interval for wireless device data from Kismet. Integration is currently configured to report devices seen as active by any Kismet sensor within the last 600 seconds of polling. + description: Poll interval for wireless device data from Kismet. Integration is currently configured to return devices seen as active by any Kismet sensor within the last 10 minutes. global: True helpLink: elastic-fleet.html advanced: True From 7124f041388310e191c30fa97336c5f8f94c278c Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Wed, 10 Apr 2024 16:13:06 -0400 Subject: [PATCH 05/63] Update ingest pipelines to match updated mappings Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/elasticsearch/files/ingest/kismet.ad_hoc | 2 +- salt/elasticsearch/files/ingest/kismet.ap | 14 +++++++------- salt/elasticsearch/files/ingest/kismet.bridged | 2 +- salt/elasticsearch/files/ingest/kismet.client | 4 ++-- salt/elasticsearch/files/ingest/kismet.common | 4 ++-- salt/elasticsearch/files/ingest/kismet.wds_ap | 4 ++-- 6 files changed, 15 insertions(+), 15 deletions(-) diff --git a/salt/elasticsearch/files/ingest/kismet.ad_hoc b/salt/elasticsearch/files/ingest/kismet.ad_hoc index 8cbc9cd2b..adfbd7901 100644 --- a/salt/elasticsearch/files/ingest/kismet.ad_hoc +++ b/salt/elasticsearch/files/ingest/kismet.ad_hoc @@ -3,7 +3,7 @@ { "rename": { "field": "message2.kismet_device_base_macaddr", - "target_field": "wireless.bssid" + "target_field": "network.wireless.bssid" } } ] diff --git a/salt/elasticsearch/files/ingest/kismet.ap b/salt/elasticsearch/files/ingest/kismet.ap index 1b8cbb80e..107f924fd 100644 --- a/salt/elasticsearch/files/ingest/kismet.ap +++ b/salt/elasticsearch/files/ingest/kismet.ap @@ -3,35 +3,35 @@ { "rename": { "field": "message2.dot11_device.dot11_device_last_beaconed_ssid_record.dot11_advertisedssid_cloaked", - "target_field": "wireless.ssid_cloaked", + "target_field": "network.wireless.ssid_cloaked", "if": "ctx?.message2?.dot11_device?.dot11_device_last_beaconed_ssid_record?.dot11_advertisedssid_cloaked != null" } }, { "rename": { "field": "message2.dot11_device.dot11_device_last_beaconed_ssid_record.dot11_advertisedssid_ssid", - "target_field": "wireless.ssid", + "target_field": "network.wireless.ssid", "if": "ctx?.message2?.dot11_device?.dot11_device_last_beaconed_ssid_record?.dot11_advertisedssid_ssid != null" } }, { "set": { - "field": "wireless.ssid", + "field": "network.wireless.ssid", "value": "Hidden", - "if": "ctx?.wireless?.ssid_cloaked != null && ctx?.wireless?.ssid_cloaked == 1" + "if": "ctx?.network?.wireless?.ssid_cloaked != null && ctx?.network?.wireless?.ssid_cloaked == 1" } }, { "rename": { "field": "message2.dot11_device.dot11_device_last_beaconed_ssid_record.dot11_advertisedssid_dot11e_channel_utilization_perc", - "target_field": "wireless.channel_utilization", + "target_field": "network.network.wireless.channel_utilization", "if": "ctx?.message2?.dot11_device?.dot11_device_last_beaconed_ssid_record?.dot11_advertisedssid_dot11e_channel_utilization_perc != null" } }, { "rename": { "field": "message2.dot11_device.dot11_device_last_bssid", - "target_field": "wireless.bssid" + "target_field": "network.wireless.bssid" } }, { @@ -39,7 +39,7 @@ "field": "message2.dot11_device.dot11_device_associated_client_map", "processor": { "append": { - "field": "wireless.associated_clients", + "field": "network.wireless.associated_clients", "value": "{{_ingest._key}}" } }, diff --git a/salt/elasticsearch/files/ingest/kismet.bridged b/salt/elasticsearch/files/ingest/kismet.bridged index 5eee3b78c..b61635e3a 100644 --- a/salt/elasticsearch/files/ingest/kismet.bridged +++ b/salt/elasticsearch/files/ingest/kismet.bridged @@ -9,7 +9,7 @@ { "rename": { "field": "message2.dot11_device.dot11_device_last_bssid", - "target_field": "wireless.bssid" + "target_field": "network.wireless.bssid" } } ] diff --git a/salt/elasticsearch/files/ingest/kismet.client b/salt/elasticsearch/files/ingest/kismet.client index 8b3d3069b..6da0a071b 100644 --- a/salt/elasticsearch/files/ingest/kismet.client +++ b/salt/elasticsearch/files/ingest/kismet.client @@ -9,7 +9,7 @@ { "rename": { "field": "message2.dot11_device.dot11_device_last_bssid", - "target_field": "wireless.last_connected_bssid", + "target_field": "network.wireless.last_connected_bssid", "if": "ctx?.message2?.dot11_device?.dot11_device_last_bssid != null" } }, @@ -18,7 +18,7 @@ "field": "message2.dot11_device.dot11_device_client_map", "processor": { "append": { - "field": "wireless.known_connected_bssid", + "field": "network.wireless.known_connected_bssid", "value": "{{_ingest._key}}" } }, diff --git a/salt/elasticsearch/files/ingest/kismet.common b/salt/elasticsearch/files/ingest/kismet.common index 95eb29b73..368e7601a 100644 --- a/salt/elasticsearch/files/ingest/kismet.common +++ b/salt/elasticsearch/files/ingest/kismet.common @@ -73,14 +73,14 @@ { "rename": { "field": "message2.kismet_device_base_channel", - "target_field": "wireless.channel", + "target_field": "network.wireless.channel", "if": "ctx?.message2?.kismet_device_base_channel != ''" } }, { "rename": { "field": "message2.kismet_device_base_frequency", - "target_field": "wireless.frequency", + "target_field": "network.wireless.frequency", "if": "ctx?.message2?.kismet_device_base_frequency != 0" } }, diff --git a/salt/elasticsearch/files/ingest/kismet.wds_ap b/salt/elasticsearch/files/ingest/kismet.wds_ap index 7f43d43fd..4d10b211b 100644 --- a/salt/elasticsearch/files/ingest/kismet.wds_ap +++ b/salt/elasticsearch/files/ingest/kismet.wds_ap @@ -3,7 +3,7 @@ { "rename": { "field": "message2.kismet_device_base_commonname", - "target_field": "wireless.bssid" + "target_field": "network.wireless.bssid" } }, { @@ -11,7 +11,7 @@ "field": "message2.dot11_device.dot11_device_associated_client_map", "processor": { "append": { - "field": "wireless.associated_clients", + "field": "network.wireless.associated_clients", "value": "{{_ingest._key}}" } }, From ed97aa4e78e69c2da298d8881f3627c4dc4e25b5 Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Thu, 11 Apr 2024 08:21:20 -0400 Subject: [PATCH 06/63] Enable Detections Adv by default --- salt/soc/defaults.yaml | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index e1d4d1726..ac1fc1993 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -2088,6 +2088,7 @@ soc: - red customEnabled: false detections: + advanced: true viewEnabled: true createLink: /detection/create eventFetchLimit: 500 @@ -2113,23 +2114,35 @@ soc: - soc_timestamp queries: - name: "All Detections" - query: "_id:*" + query: "_id:* | groupby so_detection.language | groupby so_detection.ruleset so_detection.isEnabled" + description: Show all Detections, community and custom - name: "Custom Detections" - query: "so_detection.isCommunity:false" + query: "so_detection.isCommunity:false AND NOT so_detection.ruleset: securityonion-resources" + description: Show all custom detections - name: "All Detections - Enabled" - query: "so_detection.isEnabled:true" + query: "so_detection.isEnabled:true | groupby so_detection.language | groupby so_detection.ruleset so_detection.severity" + description: Show all enalbed Detections - name: "All Detections - Disabled" - query: "so_detection.isEnabled:false" + query: "so_detection.isEnabled:false | groupby so_detection.language | groupby so_detection.ruleset so_detection.severity" + description: Show all disabled Detections - name: "Detection Type - Suricata (NIDS)" - query: "so_detection.language:suricata" + query: "so_detection.language:suricata | groupby so_detection.ruleset so_detection.isEnabled" + description: Show all NIDS Detections, which are run with Suricata - name: "Detection Type - Sigma (Elastalert) - All" - query: "so_detection.language:sigma" + query: "so_detection.language:sigma | groupby so_detection.ruleset so_detection.isEnabled" + description: Show all Sigma Detections, which are run with Elastalert - name: "Detection Type - Sigma (Elastalert) - Windows" - query: 'so_detection.language:sigma AND so_detection.content: "*product: windows*"' + query: 'so_detection.language:sigma AND so_detection.content: "*product: windows*" | groupby so_detection.ruleset so_detection.isEnabled' + description: Show all Sigma Detections with a logsource of Windows - name: "Detection Type - YARA (Strelka)" - query: "so_detection.language:yara" + query: "so_detection.language:yara | groupby so_detection.ruleset so_detection.isEnabled" + description: Show all YARA detections, which are used by Strelka - name: "Security Onion - Grid Detections" query: "so_detection.ruleset:securityonion-resources" + description: Show Detections for this Security Onion Grid + - name: "Detections with Overrides" + query: "_exists_:so_detection.overrides | groupby so_detection.language | groupby so_detection.ruleset so_detection.isEnabled" + description: Show Detections that have Overrides detection: presets: severity: From 1c5f02ade28b0f18d2cb69675d0447cf2be0305e Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Thu, 11 Apr 2024 09:21:08 -0400 Subject: [PATCH 07/63] Update annotations --- salt/soc/soc_soc.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index b550b62c5..ed9acc47a 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -293,6 +293,7 @@ soc: alerts: *appSettings cases: *appSettings dashboards: *appSettings + detections: *appSettings grid: maxUploadSize: description: The maximum number of bytes for an uploaded PCAP import file. From fd689a4607a8f416b8cbf86905647cd8940ec47c Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Thu, 11 Apr 2024 11:18:04 -0400 Subject: [PATCH 08/63] Fix typo in ingest pipeline Test to fix duplicate events in SOC, by removing conflicting field event.created Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/elasticsearch/files/ingest/kismet.ap | 2 +- salt/elasticsearch/files/ingest/kismet.common | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/elasticsearch/files/ingest/kismet.ap b/salt/elasticsearch/files/ingest/kismet.ap index 107f924fd..a864c09e4 100644 --- a/salt/elasticsearch/files/ingest/kismet.ap +++ b/salt/elasticsearch/files/ingest/kismet.ap @@ -24,7 +24,7 @@ { "rename": { "field": "message2.dot11_device.dot11_device_last_beaconed_ssid_record.dot11_advertisedssid_dot11e_channel_utilization_perc", - "target_field": "network.network.wireless.channel_utilization", + "target_field": "network.wireless.channel_utilization", "if": "ctx?.message2?.dot11_device?.dot11_device_last_beaconed_ssid_record?.dot11_advertisedssid_dot11e_channel_utilization_perc != null" } }, diff --git a/salt/elasticsearch/files/ingest/kismet.common b/salt/elasticsearch/files/ingest/kismet.common index 368e7601a..14d439105 100644 --- a/salt/elasticsearch/files/ingest/kismet.common +++ b/salt/elasticsearch/files/ingest/kismet.common @@ -149,7 +149,8 @@ "device_type", "wifi", "agent", - "host" + "host", + "event.created" ], "ignore_failure": true } From 68e016090b2f456d62d280ffb4441ac47f62c2d0 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Thu, 11 Apr 2024 13:21:54 -0400 Subject: [PATCH 09/63] Fix network.wireless.ssid not parsing Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/elasticsearch/templates/component/ecs/network.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/templates/component/ecs/network.json b/salt/elasticsearch/templates/component/ecs/network.json index cc0f9d288..8cc6bdc37 100644 --- a/salt/elasticsearch/templates/component/ecs/network.json +++ b/salt/elasticsearch/templates/component/ecs/network.json @@ -103,7 +103,7 @@ "type": "keyword" }, "ssid_cloaked": { - "type": "boolean" + "type": "integer" }, "known_connected_bssid": { "ignore_above": 1024, From c269fb90acc4b56cfb2814105ec0e746de9808d6 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Thu, 11 Apr 2024 14:41:54 -0400 Subject: [PATCH 10/63] Added a Kismet Wifi devices dashboard for an overview of kismet data Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/soc/defaults.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 987011c99..0826f4957 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1180,6 +1180,13 @@ soc: - soc_timestamp - event.dataset - message + ':kismet:': + - soc_timestamp + - device.manufacturer + - client.mac + - network.wireless.ssid + - network.wireless.bssid + - event.dataset server: bindAddress: 0.0.0.0:9822 baseUrl: / @@ -1819,6 +1826,9 @@ soc: - name: GeoIP - Source Organizations description: GeoIP tagged logs visualized by source organizations query: '* AND _exists_:source_geo.organization_name | groupby source_geo.organization_name | groupby source.ip | groupby -sankey source.ip destination.ip | groupby destination.ip | groupby destination.port | groupby source.geo.country_name | groupby event.dataset | groupby event.module' + - name: Kismet - WiFi Devices + description: WiFi devices seen by Kismet sensors + query: 'event.module: kismet | groupby network.wireless.ssid | groupby device.manufacturer | groupby -pie device.manufacturer | groupby event.dataset' job: alerts: advanced: false From 49ccd86c399296700ae5b956df0f30abd71ec36d Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Fri, 12 Apr 2024 08:35:44 -0400 Subject: [PATCH 11/63] Fix fingerprint paths --- salt/soc/defaults.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index ac1fc1993..cf51c07fd 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1258,7 +1258,7 @@ soc: elastAlertRulesFolder: /opt/sensoroni/elastalert reposFolder: /opt/sensoroni/sigma/repos rulesFingerprintFile: /opt/sensoroni/fingerprints/sigma.fingerprint - stateFilePath: /opt/so/conf/soc/fingerprints/elastalertengine.state + stateFilePath: /opt/sensoroni/fingerprints/elastalertengine.state rulesRepos: - repo: https://github.com/Security-Onion-Solutions/securityonion-resources license: Elastic-2.0 @@ -1319,7 +1319,7 @@ soc: - repo: https://github.com/Security-Onion-Solutions/securityonion-yara license: DRL yaraRulesFolder: /opt/sensoroni/yara/rules - stateFilePath: /opt/so/conf/soc/fingerprints/strelkaengine.state + stateFilePath: /opt/sensoroni/fingerprints/strelkaengine.state suricataengine: allowRegex: '' autoUpdateEnabled: true @@ -1327,7 +1327,7 @@ soc: communityRulesFile: /nsm/rules/suricata/emerging-all.rules denyRegex: '' rulesFingerprintFile: /opt/sensoroni/fingerprints/emerging-all.fingerprint - stateFilePath: /opt/so/conf/soc/fingerprints/suricataengine.state + stateFilePath: /opt/sensoroni/fingerprints/suricataengine.state client: enableReverseLookup: false docsUrl: /docs/ From f5e42e73af53b98387e7a56d2bc6571544949a4b Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Fri, 12 Apr 2024 13:30:20 -0400 Subject: [PATCH 12/63] Add docs for ruleset change --- salt/idstools/soc_idstools.yaml | 2 +- salt/soc/defaults.yaml | 6 +++--- salt/soc/soc_soc.yaml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/salt/idstools/soc_idstools.yaml b/salt/idstools/soc_idstools.yaml index 3e3a68117..0a2bf0cbf 100644 --- a/salt/idstools/soc_idstools.yaml +++ b/salt/idstools/soc_idstools.yaml @@ -9,7 +9,7 @@ idstools: forcedType: string helpLink: rules.html ruleset: - description: 'Defines the ruleset you want to run. Options are ETOPEN or ETPRO. WARNING! Changing the ruleset will remove all existing Suricata rules of the previous ruleset and their associated overrides. This removal cannot be undone.' + description: 'Defines the ruleset you want to run. Options are ETOPEN or ETPRO. Once you have changed the ruleset here, you will need to wait for the rule update to take place (every 8 hours), or you can force the update by nagivating to Detections --> Options dropdown menu --> Suricata --> Full Update. WARNING! Changing the ruleset will remove all existing Suricata rules of the previous ruleset and their associated overrides. This removal cannot be undone.' global: True regex: ETPRO\b|ETOPEN\b helpLink: rules.html diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index cf51c07fd..b8fcf0581 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1253,7 +1253,7 @@ soc: - core+critical - securityonion-resources+critical - securityonion-resources+high - communityRulesImportFrequencySeconds: 86400 + communityRulesImportFrequencySeconds: 28800 denyRegex: '' elastAlertRulesFolder: /opt/sensoroni/elastalert reposFolder: /opt/sensoroni/sigma/repos @@ -1311,7 +1311,7 @@ soc: strelkaengine: allowRegex: '' autoUpdateEnabled: true - communityRulesImportFrequencySeconds: 86400 + communityRulesImportFrequencySeconds: 28800 compileYaraPythonScriptPath: /opt/so/conf/strelka/compile_yara.py denyRegex: '' reposFolder: /opt/sensoroni/yara/repos @@ -1323,7 +1323,7 @@ soc: suricataengine: allowRegex: '' autoUpdateEnabled: true - communityRulesImportFrequencySeconds: 86400 + communityRulesImportFrequencySeconds: 28800 communityRulesFile: /nsm/rules/suricata/emerging-all.rules denyRegex: '' rulesFingerprintFile: /opt/sensoroni/fingerprints/emerging-all.fingerprint diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index ed9acc47a..42b80a3f0 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -110,7 +110,7 @@ soc: forcedType: "[]{}" helpLink: sigma.html sigmaRulePackages: - description: 'Defines the Sigma Community Ruleset you want to run. One of these (core | core+ | core++ | all ) as well as an optional Add-on (emerging_threats_addon). WARNING! Changing the ruleset will remove all existing Sigma rules of the previous ruleset and their associated overrides. This removal cannot be undone.' + description: 'Defines the Sigma Community Ruleset you want to run. One of these (core | core+ | core++ | all ) as well as an optional Add-on (emerging_threats_addon). Once you have changed the ruleset here, you will need to wait for the rule update to take place (every 8 hours), or you can force the update by nagivating to Detections --> Options dropdown menu --> Elastalert --> Full Update. WARNING! Changing the ruleset will remove all existing Sigma rules of the previous ruleset and their associated overrides. This removal cannot be undone.' global: True advanced: False helpLink: sigma.html From dbfb178556fb7b7458d195120d0496434c3aa339 Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Tue, 16 Apr 2024 12:22:53 -0400 Subject: [PATCH 13/63] Add test --- salt/soc/defaults.yaml | 2 ++ salt/soc/soc_soc.yaml | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index b8fcf0581..c39b7bd90 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1310,6 +1310,8 @@ soc: - rbac/users_roles strelkaengine: allowRegex: '' + autoEnabledYaraRules: + - securityonion-yara autoUpdateEnabled: true communityRulesImportFrequencySeconds: 28800 compileYaraPythonScriptPath: /opt/so/conf/strelka/compile_yara.py diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index 42b80a3f0..2001fb0c1 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -186,6 +186,11 @@ soc: global: True advanced: True helpLink: yara.html + autoEnabledYaraRules: + description: 'Yara rules to automatically enable on initial import. Format is $Ruleset - for example, for the default shipped ruleset: securityonion-yara' + global: True + advanced: True + helpLink: sigma.html autoUpdateEnabled: description: 'Set to true to enable automatic Internet-connected updates of the Yara rulesets. If this is an Airgap system, this setting will be overridden and set to false.' global: True From 8cc4d2668ed5f47e1e984c2e2920d80fe33e6ef9 Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Tue, 16 Apr 2024 12:52:14 -0400 Subject: [PATCH 14/63] Move compile_yara --- .../files/bin => strelka/compile_yara}/compile_yara.py | 5 +++++ salt/strelka/config.sls | 8 ++++++++ 2 files changed, 13 insertions(+) rename salt/{soc/files/bin => strelka/compile_yara}/compile_yara.py (58%) diff --git a/salt/soc/files/bin/compile_yara.py b/salt/strelka/compile_yara/compile_yara.py similarity index 58% rename from salt/soc/files/bin/compile_yara.py rename to salt/strelka/compile_yara/compile_yara.py index 43c8b1a09..b840dcab6 100644 --- a/salt/soc/files/bin/compile_yara.py +++ b/salt/strelka/compile_yara/compile_yara.py @@ -1,3 +1,8 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + import os import yara import glob diff --git a/salt/strelka/config.sls b/salt/strelka/config.sls index 929bef113..f6c66d8ff 100644 --- a/salt/strelka/config.sls +++ b/salt/strelka/config.sls @@ -14,6 +14,14 @@ strelkaconfdir: - group: 939 - makedirs: True +strelkacompileyara: + file.managed: + - name: /opt/so/conf/strelka/compile_yara.py + - source: salt://strelka/compile_yara/compile_yara.py + - user: 939 + - group: 939 + - makedirs: True + strelkarulesdir: file.directory: - name: /opt/so/conf/strelka/rules From ff284761913ed4881c9d98ba68e2022ebcdbb1e6 Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Tue, 16 Apr 2024 13:10:17 -0400 Subject: [PATCH 15/63] Fix compile_yara path --- salt/soc/defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index c39b7bd90..6a1376478 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1314,7 +1314,7 @@ soc: - securityonion-yara autoUpdateEnabled: true communityRulesImportFrequencySeconds: 28800 - compileYaraPythonScriptPath: /opt/so/conf/strelka/compile_yara.py + compileYaraPythonScriptPath: /opt/sensoroni/yara/compile_yara.py denyRegex: '' reposFolder: /opt/sensoroni/yara/repos rulesRepos: From 67a57e9df75221c45e0704b4880af4906b177d9c Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 17 Apr 2024 13:14:45 -0400 Subject: [PATCH 16/63] Update limited-analyst.json --- salt/elasticsearch/roles/limited-analyst.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/elasticsearch/roles/limited-analyst.json b/salt/elasticsearch/roles/limited-analyst.json index 9186b732e..6511e5f44 100644 --- a/salt/elasticsearch/roles/limited-analyst.json +++ b/salt/elasticsearch/roles/limited-analyst.json @@ -13,7 +13,8 @@ "monitor", "read", "read_cross_cluster", - "view_index_metadata" + "view_index_metadata", + "write" ] } ], From 8d9aae198364cda0b0e9ae9daef9e3efc9c482f9 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Thu, 18 Apr 2024 09:28:30 -0400 Subject: [PATCH 17/63] FEATURE: Add queue=True to so-checkin so that it will wait for any running states #12815 --- salt/common/tools/sbin/so-checkin | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/salt/common/tools/sbin/so-checkin b/salt/common/tools/sbin/so-checkin index db35af410..13ea9acd9 100755 --- a/salt/common/tools/sbin/so-checkin +++ b/salt/common/tools/sbin/so-checkin @@ -5,8 +5,13 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. - - . /usr/sbin/so-common -salt-call state.highstate -l info +cat << EOF + +so-checkin will run a full salt highstate to apply all salt states. If a highstate is already running, this request will be queued and so it may pause for a few minutes before you see any more output. For more information about so-checkin and salt, please see: +https://docs.securityonion.net/en/2.4/salt.html + +EOF + +salt-call state.highstate -l info queue=True From 6c6647629ca042b56ac989ee71fe7bfe10756a83 Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Thu, 18 Apr 2024 11:32:17 -0400 Subject: [PATCH 18/63] Refactor yara for compilation --- salt/strelka/backend/enabled.sls | 2 +- salt/strelka/compile_yara/compile_yara.py | 62 ++++++++++++++++++++--- salt/strelka/config.sls | 2 +- salt/strelka/defaults.yaml | 2 +- 4 files changed, 57 insertions(+), 11 deletions(-) diff --git a/salt/strelka/backend/enabled.sls b/salt/strelka/backend/enabled.sls index 0df764a6e..a626924b1 100644 --- a/salt/strelka/backend/enabled.sls +++ b/salt/strelka/backend/enabled.sls @@ -17,7 +17,7 @@ strelka_backend: - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-strelka-backend:{{ GLOBALS.so_version }} - binds: - /opt/so/conf/strelka/backend/:/etc/strelka/:ro - - /opt/so/conf/strelka/rules/:/etc/yara/:ro + - /opt/so/conf/strelka/rules/compiled/:/etc/yara/:ro {% if DOCKER.containers['so-strelka-backend'].custom_bind_mounts %} {% for BIND in DOCKER.containers['so-strelka-backend'].custom_bind_mounts %} - {{ BIND }} diff --git a/salt/strelka/compile_yara/compile_yara.py b/salt/strelka/compile_yara/compile_yara.py index b840dcab6..2138c73d1 100644 --- a/salt/strelka/compile_yara/compile_yara.py +++ b/salt/strelka/compile_yara/compile_yara.py @@ -6,14 +6,60 @@ import os import yara import glob -import sys +import json +from concurrent.futures import ThreadPoolExecutor -def compile_yara_rules(rules_dir: str) -> None: - compiled_rules_path: str = os.path.join(rules_dir, "rules.yar.compiled") - rule_files: list[str] = glob.glob(os.path.join(rules_dir, '**/*.yar'), recursive=True) +def check_syntax(rule_file): + try: + # Testing if compilation throws a syntax error, don't save the result + yara.compile(filepath=rule_file) + return (True, rule_file, None) + except yara.SyntaxError as e: + # Return the error message for logging purposes + return (False, rule_file, str(e)) - if rule_files: - rules: yara.Rules = yara.compile(filepaths={os.path.basename(f): f for f in rule_files}) - rules.save(compiled_rules_path) +def compile_yara_rules(rules_dir): + compiled_dir = os.path.join(rules_dir, "compiled") + compiled_rules_path = os.path.join(compiled_dir, "rules.compiled") + rule_files = glob.glob(os.path.join(rules_dir, '**/*.yar'), recursive=True) + files_to_compile = {} + removed_count = 0 + success_count = 0 + + # Use ThreadPoolExecutor to parallelize syntax checks + with ThreadPoolExecutor() as executor: + results = executor.map(check_syntax, rule_files) + + # Collect yara files and prepare for batch compilation + for success, rule_file, error_message in results: + if success: + files_to_compile[os.path.basename(rule_file)] = rule_file + success_count += 1 + else: + # Extract just the UUID from the rule file name + rule_id = os.path.splitext(os.path.basename(rule_file))[0] + log_entry = { + "event.module": "soc", + "event.dataset": "soc.detections", + "log.level": "error", + "error.message": error_message, + "detection_type": "yara", + "rule.uuid": rule_id, + "error.type": "runtime_status" + } + with open('/opt/sensoroni/logs/detections_runtime-status_yara.log', 'a') as log_file: + json.dump(log_entry, log_file) + log_file.write('\n') # Ensure new entries start on new lines + os.remove(rule_file) + removed_count += 1 -compile_yara_rules(sys.argv[1]) + # Compile all remaining valid rules into a single file + if files_to_compile: + compiled_rules = yara.compile(filepaths=files_to_compile) + compiled_rules.save(compiled_rules_path) + print(f"All remaining rules compiled and saved into {compiled_rules_path}") + + # Print summary of compilation results + print(f"Summary: {success_count} rules compiled successfully, {removed_count} rules removed due to errors.") + +compile_yara_rules("/opt/sensoroni/yara/rules/") \ No newline at end of file diff --git a/salt/strelka/config.sls b/salt/strelka/config.sls index f6c66d8ff..90bba58a7 100644 --- a/salt/strelka/config.sls +++ b/salt/strelka/config.sls @@ -9,7 +9,7 @@ # Strelka config strelkaconfdir: file.directory: - - name: /opt/so/conf/strelka + - name: /opt/so/conf/strelka/rules/compiled/ - user: 939 - group: 939 - makedirs: True diff --git a/salt/strelka/defaults.yaml b/salt/strelka/defaults.yaml index da259fa14..f91ad8691 100644 --- a/salt/strelka/defaults.yaml +++ b/salt/strelka/defaults.yaml @@ -563,7 +563,7 @@ strelka: options: location: '/etc/yara/' compiled: - enabled: False + enabled: True filename: "rules.compiled" store_offset: True offset_meta_key: "StrelkaHexDump" From 229a9899143b555e1527a6e3dfdfa8cdb2ff67e2 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Thu, 18 Apr 2024 11:47:01 -0400 Subject: [PATCH 19/63] Update so-elasticsearch-cluster-space-total --- .../tools/sbin_jinja/so-elasticsearch-cluster-space-total | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-cluster-space-total b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-cluster-space-total index 3faa2a7a9..b73967c26 100755 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-cluster-space-total +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-cluster-space-total @@ -40,7 +40,7 @@ fi # Iterate through the output of _cat/allocation for each node in the cluster to determine the total available space {% if GLOBALS.role == 'so-manager' %} -for i in $(/usr/sbin/so-elasticsearch-query _cat/allocation | grep -v {{ GLOBALS.manager }} | awk '{print $5}'); do +for i in $(/usr/sbin/so-elasticsearch-query _cat/allocation | grep -v "{{ GLOBALS.manager }}$" | awk '{print $5}'); do {% else %} for i in $(/usr/sbin/so-elasticsearch-query _cat/allocation | awk '{print $5}'); do {% endif %} From 406dda60511015a215ddfae08ecb885aadd17389 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Thu, 18 Apr 2024 11:48:15 -0400 Subject: [PATCH 20/63] Update so-elasticsearch-cluster-space-used --- .../tools/sbin_jinja/so-elasticsearch-cluster-space-used | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-cluster-space-used b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-cluster-space-used index 5d8a60e22..49e634853 100755 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-cluster-space-used +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-cluster-space-used @@ -13,7 +13,7 @@ TOTAL_USED_SPACE=0 # Iterate through the output of _cat/allocation for each node in the cluster to determine the total used space {% if GLOBALS.role == 'so-manager' %} # Get total disk space - disk.total -for i in $(/usr/sbin/so-elasticsearch-query _cat/allocation | grep -v {{ GLOBALS.manager }} | awk '{print $3}'); do +for i in $(/usr/sbin/so-elasticsearch-query _cat/allocation | grep -v "{{ GLOBALS.manager }}$" | awk '{print $3}'); do {% else %} # Get disk space taken up by indices - disk.indices for i in $(/usr/sbin/so-elasticsearch-query _cat/allocation | awk '{print $2}'); do From a237ef5d96568d0293f950c41c1d6347e8a948ad Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Fri, 19 Apr 2024 16:33:35 -0400 Subject: [PATCH 21/63] Update default queries --- salt/soc/defaults.yaml | 7 ++----- salt/strelka/compile_yara/compile_yara.py | 1 + 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 6a1376478..a4e7cbf4d 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -2128,14 +2128,11 @@ soc: query: "so_detection.isEnabled:false | groupby so_detection.language | groupby so_detection.ruleset so_detection.severity" description: Show all disabled Detections - name: "Detection Type - Suricata (NIDS)" - query: "so_detection.language:suricata | groupby so_detection.ruleset so_detection.isEnabled" + query: "so_detection.language:suricata | groupby so_detection.ruleset so_detection.isEnabled | groupby so_detection.category" description: Show all NIDS Detections, which are run with Suricata - name: "Detection Type - Sigma (Elastalert) - All" - query: "so_detection.language:sigma | groupby so_detection.ruleset so_detection.isEnabled" + query: "so_detection.language:sigma | groupby so_detection.ruleset so_detection.isEnabled | groupby so_detection.category | groupby so_detection.product" description: Show all Sigma Detections, which are run with Elastalert - - name: "Detection Type - Sigma (Elastalert) - Windows" - query: 'so_detection.language:sigma AND so_detection.content: "*product: windows*" | groupby so_detection.ruleset so_detection.isEnabled' - description: Show all Sigma Detections with a logsource of Windows - name: "Detection Type - YARA (Strelka)" query: "so_detection.language:yara | groupby so_detection.ruleset so_detection.isEnabled" description: Show all YARA detections, which are used by Strelka diff --git a/salt/strelka/compile_yara/compile_yara.py b/salt/strelka/compile_yara/compile_yara.py index 2138c73d1..6d88fbbde 100644 --- a/salt/strelka/compile_yara/compile_yara.py +++ b/salt/strelka/compile_yara/compile_yara.py @@ -43,6 +43,7 @@ def compile_yara_rules(rules_dir): "event.dataset": "soc.detections", "log.level": "error", "error.message": error_message, + "error.analysis": "syntax error", "detection_type": "yara", "rule.uuid": rule_id, "error.type": "runtime_status" From 58ddd55123e535bee74f25c97f85721769395548 Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Tue, 23 Apr 2024 07:28:07 -0400 Subject: [PATCH 22/63] Exclude yara runtime log --- salt/common/tools/sbin/so-log-check | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check index d54c60168..b83c98e7a 100755 --- a/salt/common/tools/sbin/so-log-check +++ b/salt/common/tools/sbin/so-log-check @@ -236,6 +236,7 @@ exclude_log "playbook.log" # Playbook is removed as of 2.4.70, logs may still be exclude_log "mysqld.log" # MySQL is removed as of 2.4.70, logs may still be on disk exclude_log "soctopus.log" # Soctopus is removed as of 2.4.70, logs may still be on disk exclude_log "agentstatus.log" # ignore this log since it tracks agents in error state +exclude_log "detections_runtime-status_yara.log" # temporarily ignore this log until Detections is more stable for log_file in $(cat /tmp/log_check_files); do status "Checking log file $log_file" From 691b02a15e8d6783280675638707a42d967785cf Mon Sep 17 00:00:00 2001 From: weslambert Date: Tue, 23 Apr 2024 10:40:09 -0400 Subject: [PATCH 23/63] Fix warm description --- salt/elasticsearch/soc_elasticsearch.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index 96a757c82..81753f16b 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -131,7 +131,7 @@ elasticsearch: helpLink: elasticsearch.html warm: min_age: - description: Minimum age of index. ex. 30d - This determines when the index should be moved to the cold tier. While still searchable, this tier is typically optimized for lower storage costs rather than search speed. + description: Minimum age of index. ex. 30d - This determines when the index should be moved to the warm tier. Nodes in the warm tier generally don’t need to be as fast as those in the hot tier. regex: ^\[0-9\]{1,5}d$ forcedType: string global: True From e53e7768a070f4f4abf31977df75054fa552eb41 Mon Sep 17 00:00:00 2001 From: Pete Date: Tue, 23 Apr 2024 21:24:39 +0000 Subject: [PATCH 24/63] check status before stopping service resolves #12811 so-verify detects rare false error If salt is uninstalled during call to so-setup where it detects a previous install, the "Failed" keyword from "systemctl stop $service" causes so-verify to falsely detect an installation error. This might happen if the user removes the salt packages between calls to so-setup, or if upgrading from Ubuntu 20.04 to 22.04 then installing 2.4.xx on top of a 2.3.xx installation. The fix is to wrap the call to stop the service in a check if the service is running. This ignores the setting of pid var, as the next use of pid is within a while loop that will not execute for the same reason the systemctl stop call was not launched in the background. --- setup/so-functions | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index d19f27620..b60989fb2 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1603,7 +1603,9 @@ reinstall_init() { # Kill any salt processes (safely) for service in "${salt_services[@]}"; do # Stop the service in the background so we can exit after a certain amount of time - systemctl stop "$service" & + if check_service_status "$service"; then + systemctl stop "$service" & + fi local pid=$! local count=0 From 8a0a4357005875d14aef7350a3828f76d2cc5eff Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 24 Apr 2024 08:35:19 -0400 Subject: [PATCH 25/63] Fix warm description --- salt/elasticsearch/soc_elasticsearch.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index 81753f16b..81070176c 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -271,7 +271,7 @@ elasticsearch: helpLink: elasticsearch.html warm: min_age: - description: Minimum age of index. This determines when the index should be moved to the hot tier. + description: Minimum age of index. This determines when the index should be moved to the warm tier. global: True advanced: True helpLink: elasticsearch.html From 75b5e16696a46d805b2ad76f10383f422600b517 Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 24 Apr 2024 09:14:39 -0400 Subject: [PATCH 26/63] Update description, type, and regex --- salt/elasticsearch/soc_elasticsearch.yaml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index 81070176c..dac2614ce 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -100,6 +100,8 @@ elasticsearch: hot: max_age: description: Maximum age of index. ex. 7d - This determines when the index should be moved out of the hot tier. + regex: ^\[0-9\]{1,5}d$ + forcedType: string global: True helpLink: elasticsearch.html actions: @@ -121,6 +123,8 @@ elasticsearch: cold: min_age: description: Minimum age of index. ex. 30d - This determines when the index should be moved to the cold tier. While still searchable, this tier is typically optimized for lower storage costs rather than search speed. + regex: ^\[0-9\]{1,5}d$ + forcedType: string global: True helpLink: elasticsearch.html actions: @@ -145,6 +149,8 @@ elasticsearch: delete: min_age: description: Minimum age of index. ex. 90d - This determines when the index should be deleted. + regex: ^\[0-9\]{1,5}d$ + forcedType: string global: True helpLink: elasticsearch.html so-logs: &indexSettings @@ -271,7 +277,9 @@ elasticsearch: helpLink: elasticsearch.html warm: min_age: - description: Minimum age of index. This determines when the index should be moved to the warm tier. + description: Minimum age of index. ex. 30d - This determines when the index should be moved to the warm tier. Nodes in the warm tier generally don’t need to be as fast as those in the hot tier. + regex: ^\[0-9\]{1,5}d$ + forcedType: string global: True advanced: True helpLink: elasticsearch.html @@ -296,7 +304,9 @@ elasticsearch: helpLink: elasticsearch.html cold: min_age: - description: Minimum age of index. This determines when the index should be moved to the cold tier. While still searchable, this tier is typically optimized for lower storage costs rather than search speed. + description: Minimum age of index. ex. 30d - This determines when the index should be moved to the cold tier. While still searchable, this tier is typically optimized for lower storage costs rather than search speed. + regex: ^\[0-9\]{1,5}d$ + forcedType: string global: True advanced: True helpLink: elasticsearch.html @@ -311,6 +321,8 @@ elasticsearch: delete: min_age: description: Minimum age of index. This determines when the index should be deleted. + regex: ^\[0-9\]{1,5}d$ + forcedType: string global: True advanced: True helpLink: elasticsearch.html From 1b3a0a3de8d57166e3e0228d8562d3269d316bf3 Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 24 Apr 2024 10:11:02 -0400 Subject: [PATCH 27/63] Remove hot max_age --- salt/elasticsearch/soc_elasticsearch.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index dac2614ce..7a6885075 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -98,12 +98,6 @@ elasticsearch: policy: phases: hot: - max_age: - description: Maximum age of index. ex. 7d - This determines when the index should be moved out of the hot tier. - regex: ^\[0-9\]{1,5}d$ - forcedType: string - global: True - helpLink: elasticsearch.html actions: set_priority: priority: From 0bd0c7b1ec09b86b460cf16d24b5d172bd9beaa4 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 24 Apr 2024 13:26:25 -0400 Subject: [PATCH 28/63] allow for mmap-locked to be configured --- salt/suricata/defaults.yaml | 1 + salt/suricata/map.jinja | 1 + salt/suricata/soc_suricata.yaml | 5 +++++ 3 files changed, 7 insertions(+) diff --git a/salt/suricata/defaults.yaml b/salt/suricata/defaults.yaml index 914c045b1..fa863473a 100644 --- a/salt/suricata/defaults.yaml +++ b/salt/suricata/defaults.yaml @@ -30,6 +30,7 @@ suricata: cluster-type: cluster_flow defrag: "yes" use-mmap: "yes" + mmap-locked: "yes" threads: 1 tpacket-v3: "yes" ring-size: 5000 diff --git a/salt/suricata/map.jinja b/salt/suricata/map.jinja index 2a3adf5f1..d9748acee 100644 --- a/salt/suricata/map.jinja +++ b/salt/suricata/map.jinja @@ -34,6 +34,7 @@ cluster-type: {{ SURICATAMERGED.config['af-packet']['cluster-type'] }} defrag: "{{ SURICATAMERGED.config['af-packet'].defrag }}" use-mmap: "{{ SURICATAMERGED.config['af-packet']['use-mmap'] }}" + mmap-locked: {{ SURICATAMERGED.config['af-packet']['mmap-locked'] }} threads: {{ SURICATAMERGED.config['af-packet'].threads }} tpacket-v3: "{{ SURICATAMERGED.config['af-packet']['tpacket-v3'] }}" ring-size: {{ SURICATAMERGED.config['af-packet']['ring-size'] }} diff --git a/salt/suricata/soc_suricata.yaml b/salt/suricata/soc_suricata.yaml index b0a864329..a1847167c 100644 --- a/salt/suricata/soc_suricata.yaml +++ b/salt/suricata/soc_suricata.yaml @@ -83,6 +83,11 @@ suricata: use-mmap: advanced: True readonly: True + mmap-locked: + description: Prevent swapping by locking the memory map. + advanced: True + regex: ^(yes|no)$ + helpLink: suricata.html threads: description: The amount of worker threads. helpLink: suricata.html From 4b7f826a2a8ca44f5f6eb7d809b50fb09d2ba1c8 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 24 Apr 2024 13:29:55 -0400 Subject: [PATCH 29/63] quote is so true becomes yes --- salt/suricata/map.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/suricata/map.jinja b/salt/suricata/map.jinja index d9748acee..a5012317a 100644 --- a/salt/suricata/map.jinja +++ b/salt/suricata/map.jinja @@ -34,7 +34,7 @@ cluster-type: {{ SURICATAMERGED.config['af-packet']['cluster-type'] }} defrag: "{{ SURICATAMERGED.config['af-packet'].defrag }}" use-mmap: "{{ SURICATAMERGED.config['af-packet']['use-mmap'] }}" - mmap-locked: {{ SURICATAMERGED.config['af-packet']['mmap-locked'] }} + mmap-locked: "{{ SURICATAMERGED.config['af-packet']['mmap-locked'] }}" threads: {{ SURICATAMERGED.config['af-packet'].threads }} tpacket-v3: "{{ SURICATAMERGED.config['af-packet']['tpacket-v3'] }}" ring-size: {{ SURICATAMERGED.config['af-packet']['ring-size'] }} From 13a6520a8ca3ab753906f837b1d767b4e8b548ea Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 24 Apr 2024 13:50:12 -0400 Subject: [PATCH 30/63] mmap-locked default no --- salt/suricata/defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/suricata/defaults.yaml b/salt/suricata/defaults.yaml index fa863473a..d819d1cf9 100644 --- a/salt/suricata/defaults.yaml +++ b/salt/suricata/defaults.yaml @@ -30,7 +30,7 @@ suricata: cluster-type: cluster_flow defrag: "yes" use-mmap: "yes" - mmap-locked: "yes" + mmap-locked: "no" threads: 1 tpacket-v3: "yes" ring-size: 5000 From 59a02635ed9953ffc25f14f0b3f6c2263ebe872f Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 24 Apr 2024 15:18:49 -0400 Subject: [PATCH 31/63] Change index sorting --- .../so-elasticsearch-indices-delete-delete | 36 +++++++++++-------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-indices-delete-delete b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-indices-delete-delete index 07feb36bd..5577fde3f 100755 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-indices-delete-delete +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-indices-delete-delete @@ -27,6 +27,7 @@ overlimit() { # 2. Check if the maximum number of iterations - MAX_ITERATIONS - has been exceeded. If so, exit. # Closed indices will be deleted first. If we are able to bring disk space under LOG_SIZE_LIMIT, or the number of iterations has exceeded the maximum allowed number of iterations, we will break out of the loop. + while overlimit && [[ $ITERATION -lt $MAX_ITERATIONS ]]; do # If we can't query Elasticsearch, then immediately return false. @@ -34,28 +35,35 @@ while overlimit && [[ $ITERATION -lt $MAX_ITERATIONS ]]; do [ $? -eq 1 ] && echo "$(date) - Could not query Elasticsearch." >> ${LOG} && exit # We iterate through the closed and open indices - CLOSED_INDICES=$(/usr/sbin/so-elasticsearch-query _cat/indices?h=index,status | grep 'close$' | awk '{print $1}' | grep -vE "playbook|so-case" | grep -E "(logstash-|so-|.ds-logs-)" | sort -t- -k3) - OPEN_INDICES=$(/usr/sbin/so-elasticsearch-query _cat/indices?h=index,status | grep 'open$' | awk '{print $1}' | grep -vE "playbook|so-case" | grep -E "(logstash-|so-|.ds-logs-)" | sort -t- -k3) + CLOSED_SO_INDICES=$(/usr/sbin/so-elasticsearch-query _cat/indices?h=index,status | grep 'close$' | awk '{print $1}' | grep -E "(^logstash-.*|^so-.*)" | grep -vE "so-case|so-detection" | sort -t- -k3) + CLOSED_INDICES=$(/usr/sbin/so-elasticsearch-query _cat/indices?h=index,status | grep 'close$' | awk '{print $1}' | grep -E "^.ds-logs-.*" | sort -t- -k4) + OPEN_SO_INDICES=$(/usr/sbin/so-elasticsearch-query _cat/indices?h=index,status | grep 'open$' | awk '{print $1}' | grep -E "(^logstash-.*|^so-.*)" | grep -vE "so-case|so-detection" | sort -t- -k3) + OPEN_INDICES=$(/usr/sbin/so-elasticsearch-query _cat/indices?h=index,status | grep 'open$' | awk '{print $1}' | grep -E "^.ds-logs-.*" | sort -t- -k4) - for INDEX in ${CLOSED_INDICES} ${OPEN_INDICES}; do + for INDEX in ${CLOSED_SO_INDICES} ${OPEN_SO_INDICES} ${CLOSED_INDICES} ${OPEN_INDICES}; do # Now that we've sorted the indices from oldest to newest, we need to check each index to see if it is assigned as the current write index for a data stream # To do so, we need to identify to which data stream this index is associated # We extract the data stream name using the pattern below DATASTREAM_PATTERN="logs-[a-zA-Z_.]+-[a-zA-Z_.]+" DATASTREAM=$(echo "${INDEX}" | grep -oE "$DATASTREAM_PATTERN") - # We look up the data stream, and determine the write index. If there is only one backing index, we delete the entire data stream - BACKING_INDICES=$(/usr/sbin/so-elasticsearch-query _data_stream/${DATASTREAM} | jq -r '.data_streams[0].indices | length') - if [ "$BACKING_INDICES" -gt 1 ]; then - CURRENT_WRITE_INDEX=$(/usr/sbin/so-elasticsearch-query _data_stream/$DATASTREAM | jq -r .data_streams[0].indices[-1].index_name) - # We make sure we are not trying to delete a write index - if [ "${INDEX}" != "${CURRENT_WRITE_INDEX}" ]; then - # This should not be a write index, so we should be allowed to delete it - printf "\n$(date) - Used disk space exceeds LOG_SIZE_LIMIT (${LOG_SIZE_LIMIT_GB} GB) - Deleting ${INDEX} index...\n" >> ${LOG} - /usr/sbin/so-elasticsearch-query ${INDEX} -XDELETE >> ${LOG} 2>&1 - fi + if [[ "$INDEX" =~ "^so-.*" ]]; then + printf "\n$(date) - Used disk space exceeds LOG_SIZE_LIMIT (${LOG_SIZE_LIMIT_GB} GB) - Deleting ${INDEX} index...\n" >> ${LOG} + /usr/sbin/so-elasticsearch-query ${INDEX} -XDELETE >> ${LOG} 2>&1 else - printf "\n$(date) - Used disk space exceeds LOG_SIZE_LIMIT (${LOG_SIZE_LIMIT_GB} GB) - There is only one backing index (${INDEX}). Deleting ${DATASTREAM} data stream...\n" >> ${LOG} + # We look up the data stream, and determine the write index. If there is only one backing index, we delete the entire data stream + BACKING_INDICES=$(/usr/sbin/so-elasticsearch-query _data_stream/${DATASTREAM} | jq -r '.data_streams[0].indices | length') + if [ "$BACKING_INDICES" -gt 1 ]; then + CURRENT_WRITE_INDEX=$(/usr/sbin/so-elasticsearch-query _data_stream/$DATASTREAM | jq -r .data_streams[0].indices[-1].index_name) + # We make sure we are not trying to delete a write index + if [ "${INDEX}" != "${CURRENT_WRITE_INDEX}" ]; then + # This should not be a write index, so we should be allowed to delete it + printf "\n$(date) - Used disk space exceeds LOG_SIZE_LIMIT (${LOG_SIZE_LIMIT_GB} GB) - Deleting ${INDEX} index...\n" >> ${LOG} + /usr/sbin/so-elasticsearch-query ${INDEX} -XDELETE >> ${LOG} 2>&1 + fi + else + printf "\n$(date) - Used disk space exceeds LOG_SIZE_LIMIT (${LOG_SIZE_LIMIT_GB} GB) - There is only one backing index (${INDEX}). Deleting ${DATASTREAM} data stream...\n" >> ${LOG} /usr/sbin/so-elasticsearch-query _data_stream/$DATASTREAM -XDELETE >> ${LOG} 2>&1 + fi fi if ! overlimit ; then exit From 73b5bb1a75312b0c7a712c0ae1cde30f01fa26fd Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 24 Apr 2024 15:35:17 -0400 Subject: [PATCH 32/63] add memlock to so-suricata container --- salt/docker/defaults.yaml | 2 ++ salt/docker/soc_docker.yaml | 38 ++++++++++++++++++++++++++++++++++++- salt/suricata/enabled.sls | 6 ++++++ 3 files changed, 45 insertions(+), 1 deletion(-) diff --git a/salt/docker/defaults.yaml b/salt/docker/defaults.yaml index 2ceaecaa7..b1d3b4e44 100644 --- a/salt/docker/defaults.yaml +++ b/salt/docker/defaults.yaml @@ -180,6 +180,8 @@ docker: custom_bind_mounts: [] extra_hosts: [] extra_env: [] + ulimits: + - memlock=524288000 'so-zeek': final_octet: 99 custom_bind_mounts: [] diff --git a/salt/docker/soc_docker.yaml b/salt/docker/soc_docker.yaml index da078941a..08e0dccc5 100644 --- a/salt/docker/soc_docker.yaml +++ b/salt/docker/soc_docker.yaml @@ -63,5 +63,41 @@ docker: so-elastic-agent: *dockerOptions so-telegraf: *dockerOptions so-steno: *dockerOptions - so-suricata: *dockerOptions + so-suricata: + final_octet: + description: Last octet of the container IP address. + helpLink: docker.html + readonly: True + advanced: True + global: True + port_bindings: + description: List of port bindings for the container. + helpLink: docker.html + advanced: True + multiline: True + forcedType: "[]string" + custom_bind_mounts: + description: List of custom local volume bindings. + advanced: True + helpLink: docker.html + multiline: True + forcedType: "[]string" + extra_hosts: + description: List of additional host entries for the container. + advanced: True + helpLink: docker.html + multiline: True + forcedType: "[]string" + extra_env: + description: List of additional ENV entries for the container. + advanced: True + helpLink: docker.html + multiline: True + forcedType: "[]string" + ulimits: + description: Ulimits for the container, in bytes. + advanced: True + helpLink: docker.html + multiline: True + forcedType: "[]string" so-zeek: *dockerOptions diff --git a/salt/suricata/enabled.sls b/salt/suricata/enabled.sls index d35160527..8520187d0 100644 --- a/salt/suricata/enabled.sls +++ b/salt/suricata/enabled.sls @@ -24,6 +24,12 @@ so-suricata: - {{ XTRAENV }} {% endfor %} {% endif %} + {% if DOCKER.containers['so-suricata'].ulimits %} + - ulimits: + {% for ULIMIT in DOCKER.containers['so-suricata'].ulimits %} + - {{ ULIMIT }} + {% endfor %} + {% endif %} - binds: - /opt/so/conf/suricata/suricata.yaml:/etc/suricata/suricata.yaml:ro - /opt/so/conf/suricata/threshold.conf:/etc/suricata/threshold.conf:ro From c9d9979f2250491025d7779731389f8153f4d833 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 24 Apr 2024 16:18:45 -0400 Subject: [PATCH 33/63] allow for enabled/disable of so-elasticsearch-indices-delete cronjob --- salt/elasticsearch/defaults.yaml | 3 ++- salt/elasticsearch/enabled.sls | 13 ++++++++++--- salt/elasticsearch/soc_elasticsearch.yaml | 4 ++++ 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 09f3bd681..53340340f 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -1,4 +1,6 @@ elasticsearch: + enabled: false + index_clean: true config: action: destructive_requires_name: true @@ -54,7 +56,6 @@ elasticsearch: enabled: true key: /usr/share/elasticsearch/config/elasticsearch.key verification_mode: none - enabled: false pipelines: custom001: description: Custom Pipeline diff --git a/salt/elasticsearch/enabled.sls b/salt/elasticsearch/enabled.sls index b9c66f231..1d35d3505 100644 --- a/salt/elasticsearch/enabled.sls +++ b/salt/elasticsearch/enabled.sls @@ -200,9 +200,15 @@ so-elasticsearch-roles-load: - require: - docker_container: so-elasticsearch - file: elasticsearch_sbin_jinja -{% if grains.role in ['so-eval', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-manager'] %} + +{% if grains.role in ['so-eval', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-manager'] %} +{% if ELASTICSEARCHMERGED.index_clean %} +{% set ap = "present" %} +{% else %} +{% set ap = "absent" %} +{% endif %} so-elasticsearch-indices-delete: - cron.present: + cron.{{ap}}: - name: /usr/sbin/so-elasticsearch-indices-delete > /opt/so/log/elasticsearch/cron-elasticsearch-indices-delete.log 2>&1 - identifier: so-elasticsearch-indices-delete - user: root @@ -211,7 +217,8 @@ so-elasticsearch-indices-delete: - daymonth: '*' - month: '*' - dayweek: '*' -{% endif %} +{% endif %} + {% endif %} {% else %} diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index 7a6885075..92c759288 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -5,6 +5,10 @@ elasticsearch: esheap: description: Specify the memory heap size in (m)egabytes for Elasticsearch. helpLink: elasticsearch.html + index_clean: + description: Enable or disable so-elasticsearch-indices-delete cron job. + forcedType: bool + helpLink: elasticsearch.html retention: retention_pct: decription: Total percentage of space used by Elasticsearch for multi node clusters From 3c3ed8b5c516af5a775cdb332055130fd3c92ba5 Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Wed, 24 Apr 2024 16:33:47 -0400 Subject: [PATCH 34/63] Add runtime status logs --- .../soc-detections-logs.json | 35 +++++++++++++++++++ salt/soc/config.sls | 11 ++++++ salt/soc/defaults.yaml | 6 ++++ .../tools/sbin/so-detections-runtime-status | 33 +++++++++++++++++ salt/strelka/compile_yara/compile_yara.py | 14 ++++---- 5 files changed, 92 insertions(+), 7 deletions(-) create mode 100644 salt/elasticfleet/files/integrations/grid-nodes_general/soc-detections-logs.json create mode 100644 salt/soc/tools/sbin/so-detections-runtime-status diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-detections-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-detections-logs.json new file mode 100644 index 000000000..5649b481d --- /dev/null +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-detections-logs.json @@ -0,0 +1,35 @@ +{ + "policy_id": "so-grid-nodes_general", + "package": { + "name": "log", + "version": "" + }, + "name": "soc-detections-logs", + "description": "Security Onion Console - Detections Logs", + "namespace": "so", + "inputs": { + "logs-logfile": { + "enabled": true, + "streams": { + "log.logs": { + "enabled": true, + "vars": { + "paths": [ + "/opt/so/log/soc/detections_runtime-status_sigma.log", + "/opt/so/log/soc/detections_runtime-status_yara.log" + ], + "exclude_files": [], + "ignore_older": "72h", + "data_stream.dataset": "soc", + "tags": [ + "so-soc" + ], + "processors": "- decode_json_fields:\n fields: [\"message\"]\n target: \"soc\"\n process_array: true\n max_depth: 2\n add_error_key: true \n- add_fields:\n target: event\n fields:\n category: host\n module: soc\n dataset_temp: detections\n- rename:\n fields:\n - from: \"soc.fields.sourceIp\"\n to: \"source.ip\"\n - from: \"soc.fields.status\"\n to: \"http.response.status_code\"\n - from: \"soc.fields.method\"\n to: \"http.request.method\"\n - from: \"soc.fields.path\"\n to: \"url.path\"\n - from: \"soc.message\"\n to: \"event.action\"\n - from: \"soc.level\"\n to: \"log.level\"\n ignore_missing: true", + "custom": "pipeline: common" + } + } + } + } + }, + "force": true +} diff --git a/salt/soc/config.sls b/salt/soc/config.sls index 3e756f977..af34f5e7c 100644 --- a/salt/soc/config.sls +++ b/salt/soc/config.sls @@ -80,6 +80,17 @@ socmotd: - mode: 600 - template: jinja +crondetectionsruntime: + cron.present: + - name: /usr/local/bin/so-detections-runtime-status cron + - identifier: detections-runtime-status + - user: socore + - minute: '*/10' + - hour: '*' + - daymonth: '*' + - month: '*' + - dayweek: '*' + socsigmafinalpipeline: file.managed: - name: /opt/so/conf/soc/sigma_final_pipeline.yaml diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 13e2021c5..1c14e61cb 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1912,6 +1912,12 @@ soc: - name: Kismet - WiFi Devices description: WiFi devices seen by Kismet sensors query: 'event.module: kismet | groupby network.wireless.ssid | groupby device.manufacturer | groupby -pie device.manufacturer | groupby event.dataset' + - name: SOC Detections - Runtime Status + description: Runtime Status of Detections + query: 'event.dataset:soc.detections | groupby soc.detection_type soc.error_type | groupby soc.error_analysis | groupby soc.rule.name | groupby soc.error_message' + + + job: alerts: advanced: false diff --git a/salt/soc/tools/sbin/so-detections-runtime-status b/salt/soc/tools/sbin/so-detections-runtime-status new file mode 100644 index 000000000..ed3ee5800 --- /dev/null +++ b/salt/soc/tools/sbin/so-detections-runtime-status @@ -0,0 +1,33 @@ +#!/bin/bash +# +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +# Set the default output destination to stdout +output_dest="/dev/stdout" + +# If the "cron" flag is passed, change the output destination to the log file +if [ "$1" = "cron" ]; then + output_dest="/opt/so/log/soc/detections_runtime-status_sigma.log" +fi + +# Run the query and output based on the output_dest value +/sbin/so-elasticsearch-query '*:elastalert_error*/_search' -d '{"query":{"range":{"@timestamp":{"gte":"now-11m","lte":"now"}}},"size": 50}' | \ +jq --compact-output '.hits.hits[] | { + _timestamp: ._source["@timestamp"], + "rule.name": ._source.data.rule, + error_type: "runtime_status", + error_message: ._source.message, + detection_type: "sigma", + event_module: "soc", + event_dataset: "soc.detections", + error_analysis: ( + if ._source.message | contains("Unknown column [winlog.channel]") then "Target logsource never seen" + elif ._source.message | contains("parsing_exception") then "Syntax Error" + else "Unknown" + end + ) + }' >> $output_dest + diff --git a/salt/strelka/compile_yara/compile_yara.py b/salt/strelka/compile_yara/compile_yara.py index 6d88fbbde..ece3c6a9e 100644 --- a/salt/strelka/compile_yara/compile_yara.py +++ b/salt/strelka/compile_yara/compile_yara.py @@ -39,14 +39,14 @@ def compile_yara_rules(rules_dir): # Extract just the UUID from the rule file name rule_id = os.path.splitext(os.path.basename(rule_file))[0] log_entry = { - "event.module": "soc", - "event.dataset": "soc.detections", + "event_module": "soc", + "event_dataset": "soc.detections", "log.level": "error", - "error.message": error_message, - "error.analysis": "syntax error", - "detection_type": "yara", - "rule.uuid": rule_id, - "error.type": "runtime_status" + "error_message": error_message, + "error_analysis": "Syntax Error", + "detection_type": "YARA", + "rule_uuid": rule_id, + "error_type": "runtime_status" } with open('/opt/sensoroni/logs/detections_runtime-status_yara.log', 'a') as log_file: json.dump(log_entry, log_file) From ab832e4bb2fc6341c9c61c4949a0542b651aef10 Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 24 Apr 2024 17:17:53 -0400 Subject: [PATCH 35/63] Include logstash-prefixed indices --- .../tools/sbin_jinja/so-elasticsearch-indices-delete-delete | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-indices-delete-delete b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-indices-delete-delete index 5577fde3f..44f27e9d4 100755 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-indices-delete-delete +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-indices-delete-delete @@ -46,7 +46,7 @@ while overlimit && [[ $ITERATION -lt $MAX_ITERATIONS ]]; do # We extract the data stream name using the pattern below DATASTREAM_PATTERN="logs-[a-zA-Z_.]+-[a-zA-Z_.]+" DATASTREAM=$(echo "${INDEX}" | grep -oE "$DATASTREAM_PATTERN") - if [[ "$INDEX" =~ "^so-.*" ]]; then + if [[ "$INDEX" =~ "^logstash-.*|so-.*" ]]; then printf "\n$(date) - Used disk space exceeds LOG_SIZE_LIMIT (${LOG_SIZE_LIMIT_GB} GB) - Deleting ${INDEX} index...\n" >> ${LOG} /usr/sbin/so-elasticsearch-query ${INDEX} -XDELETE >> ${LOG} 2>&1 else From 44afa55274233c3c656691a35e9a4511edf9da93 Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 24 Apr 2024 17:41:37 -0400 Subject: [PATCH 36/63] Fix comments about deletion --- .../sbin_jinja/so-elasticsearch-indices-delete-delete | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-indices-delete-delete b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-indices-delete-delete index 44f27e9d4..5e97a3f19 100755 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-indices-delete-delete +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-indices-delete-delete @@ -41,15 +41,16 @@ while overlimit && [[ $ITERATION -lt $MAX_ITERATIONS ]]; do OPEN_INDICES=$(/usr/sbin/so-elasticsearch-query _cat/indices?h=index,status | grep 'open$' | awk '{print $1}' | grep -E "^.ds-logs-.*" | sort -t- -k4) for INDEX in ${CLOSED_SO_INDICES} ${OPEN_SO_INDICES} ${CLOSED_INDICES} ${OPEN_INDICES}; do - # Now that we've sorted the indices from oldest to newest, we need to check each index to see if it is assigned as the current write index for a data stream - # To do so, we need to identify to which data stream this index is associated - # We extract the data stream name using the pattern below - DATASTREAM_PATTERN="logs-[a-zA-Z_.]+-[a-zA-Z_.]+" - DATASTREAM=$(echo "${INDEX}" | grep -oE "$DATASTREAM_PATTERN") + # Check if index is an older index. If it is an older index, delete it before moving on to newer indices. if [[ "$INDEX" =~ "^logstash-.*|so-.*" ]]; then printf "\n$(date) - Used disk space exceeds LOG_SIZE_LIMIT (${LOG_SIZE_LIMIT_GB} GB) - Deleting ${INDEX} index...\n" >> ${LOG} /usr/sbin/so-elasticsearch-query ${INDEX} -XDELETE >> ${LOG} 2>&1 else + # Now that we've sorted the indices from oldest to newest, we need to check each index to see if it is assigned as the current write index for a data stream + # To do so, we need to identify to which data stream this index is associated + # We extract the data stream name using the pattern below + DATASTREAM_PATTERN="logs-[a-zA-Z_.]+-[a-zA-Z_.]+" + DATASTREAM=$(echo "${INDEX}" | grep -oE "$DATASTREAM_PATTERN") # We look up the data stream, and determine the write index. If there is only one backing index, we delete the entire data stream BACKING_INDICES=$(/usr/sbin/so-elasticsearch-query _data_stream/${DATASTREAM} | jq -r '.data_streams[0].indices | length') if [ "$BACKING_INDICES" -gt 1 ]; then From d50de804a8c756e79c9fef6b4cd8094e41c2df6f Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 25 Apr 2024 09:04:34 -0400 Subject: [PATCH 37/63] update annotation --- salt/elasticsearch/soc_elasticsearch.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index 92c759288..210697bba 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -6,7 +6,7 @@ elasticsearch: description: Specify the memory heap size in (m)egabytes for Elasticsearch. helpLink: elasticsearch.html index_clean: - description: Enable or disable so-elasticsearch-indices-delete cron job. + description: Determines if indices should be considered for deletion by available disk space in the cluster. Otherwise, indices will only be deleted by the age defined in the ILM settings. forcedType: bool helpLink: elasticsearch.html retention: From b42442629844df877b1105abbcb5109b76076ff7 Mon Sep 17 00:00:00 2001 From: weslambert Date: Thu, 25 Apr 2024 09:14:18 -0400 Subject: [PATCH 38/63] Exclude suricata --- .../tools/sbin_jinja/so-elasticsearch-indices-delete-delete | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-indices-delete-delete b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-indices-delete-delete index 5e97a3f19..a00437a25 100755 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-indices-delete-delete +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-indices-delete-delete @@ -36,9 +36,9 @@ while overlimit && [[ $ITERATION -lt $MAX_ITERATIONS ]]; do # We iterate through the closed and open indices CLOSED_SO_INDICES=$(/usr/sbin/so-elasticsearch-query _cat/indices?h=index,status | grep 'close$' | awk '{print $1}' | grep -E "(^logstash-.*|^so-.*)" | grep -vE "so-case|so-detection" | sort -t- -k3) - CLOSED_INDICES=$(/usr/sbin/so-elasticsearch-query _cat/indices?h=index,status | grep 'close$' | awk '{print $1}' | grep -E "^.ds-logs-.*" | sort -t- -k4) + CLOSED_INDICES=$(/usr/sbin/so-elasticsearch-query _cat/indices?h=index,status | grep 'close$' | awk '{print $1}' | grep -E "^.ds-logs-.*" | grep -v "suricata" | sort -t- -k4) OPEN_SO_INDICES=$(/usr/sbin/so-elasticsearch-query _cat/indices?h=index,status | grep 'open$' | awk '{print $1}' | grep -E "(^logstash-.*|^so-.*)" | grep -vE "so-case|so-detection" | sort -t- -k3) - OPEN_INDICES=$(/usr/sbin/so-elasticsearch-query _cat/indices?h=index,status | grep 'open$' | awk '{print $1}' | grep -E "^.ds-logs-.*" | sort -t- -k4) + OPEN_INDICES=$(/usr/sbin/so-elasticsearch-query _cat/indices?h=index,status | grep 'open$' | awk '{print $1}' | grep -E "^.ds-logs-.*" | grep -v "suricata" | sort -t- -k4) for INDEX in ${CLOSED_SO_INDICES} ${OPEN_SO_INDICES} ${CLOSED_INDICES} ${OPEN_INDICES}; do # Check if index is an older index. If it is an older index, delete it before moving on to newer indices. From 2c7eb3c755ef475e082a59645ba2e98c3abcddd0 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 25 Apr 2024 10:05:59 -0400 Subject: [PATCH 39/63] only apply ulimits to suricata container if user enable mmap-locked --- salt/suricata/enabled.sls | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/salt/suricata/enabled.sls b/salt/suricata/enabled.sls index 8520187d0..3e015d100 100644 --- a/salt/suricata/enabled.sls +++ b/salt/suricata/enabled.sls @@ -7,6 +7,7 @@ {% if sls.split('.')[0] in allowed_states %} {% from 'vars/globals.map.jinja' import GLOBALS %} {% from 'docker/docker.map.jinja' import DOCKER %} +{% from 'suricata/map.jinja' import SURICATAMERGED %} include: @@ -24,7 +25,8 @@ so-suricata: - {{ XTRAENV }} {% endfor %} {% endif %} - {% if DOCKER.containers['so-suricata'].ulimits %} + {# we look at SURICATAMERGED.config['af-packet'][0] since we only allow one interface and therefore always the first list item #} + {% if SURICATAMERGED.config['af-packet'][0]['mmap-locked'] == "yes" and DOCKER.containers['so-suricata'].ulimits %} - ulimits: {% for ULIMIT in DOCKER.containers['so-suricata'].ulimits %} - {{ ULIMIT }} From 89cb8b79fdad0d1fdcb46c09bb4f6cd322cdeed7 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 29 Apr 2024 08:07:19 -0400 Subject: [PATCH 40/63] restrict workflows to so --- .github/workflows/close-threads.yml | 1 + .github/workflows/lock-threads.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/close-threads.yml b/.github/workflows/close-threads.yml index 059a35a9f..41b267f2b 100644 --- a/.github/workflows/close-threads.yml +++ b/.github/workflows/close-threads.yml @@ -15,6 +15,7 @@ concurrency: jobs: close-threads: + if: github.repository_owner == 'security-onion-solutions' runs-on: ubuntu-latest permissions: issues: write diff --git a/.github/workflows/lock-threads.yml b/.github/workflows/lock-threads.yml index eeaa444ed..f5d46ed46 100644 --- a/.github/workflows/lock-threads.yml +++ b/.github/workflows/lock-threads.yml @@ -15,6 +15,7 @@ concurrency: jobs: lock-threads: + if: github.repository_owner == 'security-onion-solutions' runs-on: ubuntu-latest steps: - uses: jertel/lock-threads@main From f2c3c928fc4753fba8888110f2470fd5c7e40bea Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Mon, 29 Apr 2024 08:49:05 -0400 Subject: [PATCH 41/63] Sigma pivot fix and cleanup --- .../files/modules/so/playbook-es.py | 38 ----------- .../files/modules/so/securityonion-es.py | 63 +++++++++++++++++++ salt/soc/defaults.yaml | 1 + 3 files changed, 64 insertions(+), 38 deletions(-) delete mode 100644 salt/elastalert/files/modules/so/playbook-es.py create mode 100644 salt/elastalert/files/modules/so/securityonion-es.py diff --git a/salt/elastalert/files/modules/so/playbook-es.py b/salt/elastalert/files/modules/so/playbook-es.py deleted file mode 100644 index 3a43c26c1..000000000 --- a/salt/elastalert/files/modules/so/playbook-es.py +++ /dev/null @@ -1,38 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - - -from time import gmtime, strftime -import requests,json -from elastalert.alerts import Alerter - -import urllib3 -urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - -class PlaybookESAlerter(Alerter): - """ - Use matched data to create alerts in elasticsearch - """ - - required_options = set(['play_title','play_url','sigma_level']) - - def alert(self, matches): - for match in matches: - today = strftime("%Y.%m.%d", gmtime()) - timestamp = strftime("%Y-%m-%d"'T'"%H:%M:%S"'.000Z', gmtime()) - headers = {"Content-Type": "application/json"} - - creds = None - if 'es_username' in self.rule and 'es_password' in self.rule: - creds = (self.rule['es_username'], self.rule['es_password']) - - payload = {"tags":"alert","rule": { "name": self.rule['play_title'],"case_template": self.rule['play_id'],"uuid": self.rule['play_id'],"category": self.rule['rule.category']},"event":{ "severity": self.rule['event.severity'],"module": self.rule['event.module'],"dataset": self.rule['event.dataset'],"severity_label": self.rule['sigma_level']},"kibana_pivot": self.rule['kibana_pivot'],"soc_pivot": self.rule['soc_pivot'],"play_url": self.rule['play_url'],"sigma_level": self.rule['sigma_level'],"event_data": match, "@timestamp": timestamp} - url = f"https://{self.rule['es_host']}:{self.rule['es_port']}/logs-playbook.alerts-so/_doc/" - requests.post(url, data=json.dumps(payload), headers=headers, verify=False, auth=creds) - - def get_info(self): - return {'type': 'PlaybookESAlerter'} diff --git a/salt/elastalert/files/modules/so/securityonion-es.py b/salt/elastalert/files/modules/so/securityonion-es.py new file mode 100644 index 000000000..0a82bdce6 --- /dev/null +++ b/salt/elastalert/files/modules/so/securityonion-es.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- + +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + + +from time import gmtime, strftime +import requests,json +from elastalert.alerts import Alerter + +import urllib3 +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + +class SecurityOnionESAlerter(Alerter): + """ + Use matched data to create alerts in Elasticsearch. + """ + + required_options = set(['detection_title', 'sigma_level']) + optional_fields = ['sigma_category', 'sigma_product', 'sigma_service'] + + def alert(self, matches): + for match in matches: + timestamp = strftime("%Y-%m-%d"'T'"%H:%M:%S"'.000Z', gmtime()) + headers = {"Content-Type": "application/json"} + + creds = None + if 'es_username' in self.rule and 'es_password' in self.rule: + creds = (self.rule['es_username'], self.rule['es_password']) + + # Start building the rule dict + rule_info = { + "name": self.rule['detection_title'], + "uuid": self.rule['detection_public_id'] + } + + # Add optional fields if they are present in the rule + for field in self.optional_fields: + rule_key = field.split('_')[-1] # Assumes field format "sigma_" + if field in self.rule: + rule_info[rule_key] = self.rule[field] + + # Construct the payload with the conditional rule_info + payload = { + "tags": "alert", + "rule": rule_info, + "event": { + "severity": self.rule['event.severity'], + "module": self.rule['event.module'], + "dataset": self.rule['event.dataset'], + "severity_label": self.rule['sigma_level'] + }, + "sigma_level": self.rule['sigma_level'], + "event_data": match, + "@timestamp": timestamp + } + url = f"https://{self.rule['es_host']}:{self.rule['es_port']}/logs-playbook.alerts-so/_doc/" + requests.post(url, data=json.dumps(payload), headers=headers, verify=False, auth=creds) + + def get_info(self): + return {'type': 'SecurityOnionESAlerter'} \ No newline at end of file diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 1c14e61cb..9be17bcca 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1961,6 +1961,7 @@ soc: - rule.name - event.severity_label - event_data.event.dataset + - rule.category - event_data.source.ip - event_data.source.port - event_data.destination.host From a663bf63c61a5f51c53b7116a067a3fa94bc03de Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 29 Apr 2024 14:22:04 -0400 Subject: [PATCH 42/63] set Suricata as default pcap engine for eval --- setup/so-functions | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/setup/so-functions b/setup/so-functions index e49d0dbea..60908e0d4 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1330,6 +1330,10 @@ create_global() { echo " influxdb_host: '$HOSTNAME'" >> $global_pillar_file echo " registry_host: '$HOSTNAME'" >> $global_pillar_file echo " endgamehost: '$ENDGAMEHOST'" >> $global_pillar_file + + if [ "$install_type" = 'EVAL' ]; then + echo " pcapengine: SURICATA" >> $global_pillar_file + fi } create_sensoroni_pillar() { From 192d91565dc1f4b7a1ebcb1597026485a9361583 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 29 Apr 2024 16:34:29 -0400 Subject: [PATCH 43/63] Update final pipeline timestamp format for event.module system events Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/elasticsearch/files/ingest/.fleet_final_pipeline-1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/files/ingest/.fleet_final_pipeline-1 b/salt/elasticsearch/files/ingest/.fleet_final_pipeline-1 index 89216077a..524d56071 100644 --- a/salt/elasticsearch/files/ingest/.fleet_final_pipeline-1 +++ b/salt/elasticsearch/files/ingest/.fleet_final_pipeline-1 @@ -80,7 +80,7 @@ { "set": { "if": "ctx.network?.type == 'ipv6'", "override": true, "field": "destination.ipv6", "value": "true" } }, { "set": { "if": "ctx.tags.0 == 'import'", "override": true, "field": "data_stream.dataset", "value": "import" } }, { "set": { "if": "ctx.tags.0 == 'import'", "override": true, "field": "data_stream.namespace", "value": "so" } }, - { "date": { "if": "ctx.event?.module == 'system'", "field": "event.created", "target_field": "@timestamp", "formats": ["yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'"] } }, + { "date": { "if": "ctx.event?.module == 'system'", "field": "event.created", "target_field": "@timestamp", "formats": ["yyyy-MM-dd'T'HH:mm:ss.SSSX"] } }, { "community_id":{ "if": "ctx.event?.dataset == 'endpoint.events.network'", "ignore_failure":true } }, { "set": { "if": "ctx.event?.module == 'fim'", "override": true, "field": "event.module", "value": "file_integrity" } }, { "rename": { "if": "ctx.winlog?.provider_name == 'Microsoft-Windows-Windows Defender'", "ignore_missing": true, "field": "winlog.event_data.Threat Name", "target_field": "winlog.event_data.threat_name" } }, From fadb6e2aa9cae3278f384b6e16016513f928e9fa Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 29 Apr 2024 16:57:48 -0400 Subject: [PATCH 44/63] Re-add original timestamp format + ignore failures with this processor Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/elasticsearch/files/ingest/.fleet_final_pipeline-1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/files/ingest/.fleet_final_pipeline-1 b/salt/elasticsearch/files/ingest/.fleet_final_pipeline-1 index 524d56071..c3e70ec2c 100644 --- a/salt/elasticsearch/files/ingest/.fleet_final_pipeline-1 +++ b/salt/elasticsearch/files/ingest/.fleet_final_pipeline-1 @@ -80,7 +80,7 @@ { "set": { "if": "ctx.network?.type == 'ipv6'", "override": true, "field": "destination.ipv6", "value": "true" } }, { "set": { "if": "ctx.tags.0 == 'import'", "override": true, "field": "data_stream.dataset", "value": "import" } }, { "set": { "if": "ctx.tags.0 == 'import'", "override": true, "field": "data_stream.namespace", "value": "so" } }, - { "date": { "if": "ctx.event?.module == 'system'", "field": "event.created", "target_field": "@timestamp", "formats": ["yyyy-MM-dd'T'HH:mm:ss.SSSX"] } }, + { "date": { "if": "ctx.event?.module == 'system'", "field": "event.created", "target_field": "@timestamp","ignore_failure": true, "formats": ["yyyy-MM-dd'T'HH:mm:ss.SSSX","yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'"] } }, { "community_id":{ "if": "ctx.event?.dataset == 'endpoint.events.network'", "ignore_failure":true } }, { "set": { "if": "ctx.event?.module == 'fim'", "override": true, "field": "event.module", "value": "file_integrity" } }, { "rename": { "if": "ctx.winlog?.provider_name == 'Microsoft-Windows-Windows Defender'", "ignore_missing": true, "field": "winlog.event_data.Threat Name", "target_field": "winlog.event_data.threat_name" } }, From ddf662bdb434142a6631572f69825db5c5974a37 Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Mon, 29 Apr 2024 16:22:30 -0600 Subject: [PATCH 45/63] Mark Repos as Community Indicate that detection rules pulled from configured repos should be marked as Community rules. --- salt/soc/defaults.yaml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 9be17bcca..051d35541 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -580,7 +580,7 @@ soc: - file.source - file.mime_type - log.id.fuid - - event.dataset + - event.dataset ':suricata:': - soc_timestamp - source.ip @@ -1270,6 +1270,7 @@ soc: - repo: https://github.com/Security-Onion-Solutions/securityonion-resources license: Elastic-2.0 folder: sigma/stable + community: true sigmaRulePackages: - core - emerging_threats_addon @@ -1327,6 +1328,7 @@ soc: rulesRepos: - repo: https://github.com/Security-Onion-Solutions/securityonion-yara license: DRL + community: true yaraRulesFolder: /opt/sensoroni/yara/rules stateFilePath: /opt/sensoroni/fingerprints/strelkaengine.state suricataengine: @@ -1917,7 +1919,7 @@ soc: query: 'event.dataset:soc.detections | groupby soc.detection_type soc.error_type | groupby soc.error_analysis | groupby soc.rule.name | groupby soc.error_message' - + job: alerts: advanced: false @@ -1955,7 +1957,7 @@ soc: - event_data.destination.host - event_data.destination.port - event_data.process.executable - - event_data.process.pid + - event_data.process.pid ':sigma:': - soc_timestamp - rule.name @@ -1967,7 +1969,7 @@ soc: - event_data.destination.host - event_data.destination.port - event_data.process.executable - - event_data.process.pid + - event_data.process.pid ':strelka:': - soc_timestamp - file.name From 4d6124f982ccf63396a2020558db4601a642d6aa Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Tue, 30 Apr 2024 10:18:34 -0400 Subject: [PATCH 46/63] FIX: Elasticsearch min_age regex #12885 --- salt/elasticsearch/soc_elasticsearch.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index 210697bba..42262a178 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -121,7 +121,7 @@ elasticsearch: cold: min_age: description: Minimum age of index. ex. 30d - This determines when the index should be moved to the cold tier. While still searchable, this tier is typically optimized for lower storage costs rather than search speed. - regex: ^\[0-9\]{1,5}d$ + regex: ^[0-9]{1,5}d$ forcedType: string global: True helpLink: elasticsearch.html @@ -134,7 +134,7 @@ elasticsearch: warm: min_age: description: Minimum age of index. ex. 30d - This determines when the index should be moved to the warm tier. Nodes in the warm tier generally don’t need to be as fast as those in the hot tier. - regex: ^\[0-9\]{1,5}d$ + regex: ^[0-9]{1,5}d$ forcedType: string global: True actions: @@ -147,7 +147,7 @@ elasticsearch: delete: min_age: description: Minimum age of index. ex. 90d - This determines when the index should be deleted. - regex: ^\[0-9\]{1,5}d$ + regex: ^[0-9]{1,5}d$ forcedType: string global: True helpLink: elasticsearch.html @@ -276,7 +276,7 @@ elasticsearch: warm: min_age: description: Minimum age of index. ex. 30d - This determines when the index should be moved to the warm tier. Nodes in the warm tier generally don’t need to be as fast as those in the hot tier. - regex: ^\[0-9\]{1,5}d$ + regex: ^[0-9]{1,5}d$ forcedType: string global: True advanced: True @@ -303,7 +303,7 @@ elasticsearch: cold: min_age: description: Minimum age of index. ex. 30d - This determines when the index should be moved to the cold tier. While still searchable, this tier is typically optimized for lower storage costs rather than search speed. - regex: ^\[0-9\]{1,5}d$ + regex: ^[0-9]{1,5}d$ forcedType: string global: True advanced: True @@ -319,7 +319,7 @@ elasticsearch: delete: min_age: description: Minimum age of index. This determines when the index should be deleted. - regex: ^\[0-9\]{1,5}d$ + regex: ^[0-9]{1,5}d$ forcedType: string global: True advanced: True From 84db82852c7825f5c0d145966e19b9496963f158 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 30 Apr 2024 15:14:56 -0400 Subject: [PATCH 47/63] annotation updates for custom settings --- salt/firewall/soc_firewall.yaml | 7 +++ salt/logstash/soc_logstash.yaml | 3 + salt/suricata/soc_suricata.yaml | 102 +++++++++----------------------- salt/zeek/soc_zeek.yaml | 4 +- 4 files changed, 42 insertions(+), 74 deletions(-) diff --git a/salt/firewall/soc_firewall.yaml b/salt/firewall/soc_firewall.yaml index 522684e07..69093dee7 100644 --- a/salt/firewall/soc_firewall.yaml +++ b/salt/firewall/soc_firewall.yaml @@ -7,6 +7,7 @@ firewall: multiline: True regex: ^(([0-9]{1,3}\.){3}[0-9]{1,3}(\/([0-9]|[1-2][0-9]|3[0-2]))?)?$ regexFailureMessage: You must enter a valid IP address or CIDR. + duplicates: True anywhere: &hostgroupsettingsadv description: List of IP or CIDR blocks to allow access to this hostgroup. forcedType: "[]string" @@ -15,6 +16,7 @@ firewall: advanced: True regex: ^(([0-9]{1,3}\.){3}[0-9]{1,3}(\/([0-9]|[1-2][0-9]|3[0-2]))?)?$ regexFailureMessage: You must enter a valid IP address or CIDR. + duplicates: True beats_endpoint: *hostgroupsettings beats_endpoint_ssl: *hostgroupsettings dockernet: &ROhostgroupsettingsadv @@ -53,6 +55,7 @@ firewall: multiline: True regex: ^(([0-9]{1,3}\.){3}[0-9]{1,3}(\/([0-9]|[1-2][0-9]|3[0-2]))?)?$ regexFailureMessage: You must enter a valid IP address or CIDR. + duplicates: True customhostgroup1: *customhostgroupsettings customhostgroup2: *customhostgroupsettings customhostgroup3: *customhostgroupsettings @@ -70,12 +73,14 @@ firewall: helpLink: firewall.html advanced: True multiline: True + duplicates: True udp: &udpsettings description: List of UDP ports for this port group. forcedType: "[]string" helpLink: firewall.html advanced: True multiline: True + duplicates: True agrules: tcp: *tcpsettings udp: *udpsettings @@ -187,6 +192,7 @@ firewall: multiline: True forcedType: "[]string" helpLink: firewall.html + duplicates: True sensor: portgroups: *portgroupsdocker searchnode: @@ -240,6 +246,7 @@ firewall: multiline: True forcedType: "[]string" helpLink: firewall.html + duplicates: True dockernet: portgroups: *portgroupshost localhost: diff --git a/salt/logstash/soc_logstash.yaml b/salt/logstash/soc_logstash.yaml index 3172ff7c5..cc81d3103 100644 --- a/salt/logstash/soc_logstash.yaml +++ b/salt/logstash/soc_logstash.yaml @@ -10,6 +10,7 @@ logstash: helpLink: logstash.html multiline: True forcedType: "[]string" + duplicates: True receiver: *assigned_pipelines heavynode: *assigned_pipelines searchnode: *assigned_pipelines @@ -23,6 +24,7 @@ logstash: helpLink: logstash.html multiline: True forcedType: "[]string" + duplicates: True fleet: *defined_pipelines manager: *defined_pipelines search: *defined_pipelines @@ -38,6 +40,7 @@ logstash: multiline: True forcedType: string helpLink: logstash.html + duplicates: True custom002: *pipeline_config custom003: *pipeline_config custom004: *pipeline_config diff --git a/salt/suricata/soc_suricata.yaml b/salt/suricata/soc_suricata.yaml index a1847167c..78c28f9e4 100644 --- a/salt/suricata/soc_suricata.yaml +++ b/salt/suricata/soc_suricata.yaml @@ -148,84 +148,40 @@ suricata: helpLink: suricata.html vars: address-groups: - HOME_NET: - description: List of hosts or networks. + HOME_NET: &suriaddressgroup + description: Assign a list of hosts, or networks, using CIDR notation, to this Suricata variable. The variable can then be re-used within Suricata rules. This allows for a single adjustment to the variable that will then affect all rules referencing the variable. regex: ^(((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\/([0-9]|[1-2][0-9]|3[0-2]))?$|^((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?))|:))|(([0-9A-Fa-f]{1,4}:){5}((:[0-9A-Fa-f]{1,4}){1,2}|:((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)|:))|(([0-9A-Fa-f]{1,4}:){4}((:[0-9A-Fa-f]{1,4}){1,3}|:((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)|:))|(([0-9A-Fa-f]{1,4}:){3}((:[0-9A-Fa-f]{1,4}){1,4}|:((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)|:))|(([0-9A-Fa-f]{1,4}:){2}((:[0-9A-Fa-f]{1,4}){1,5}|:((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)|:))|(([0-9A-Fa-f]{1,4}:){1}((:[0-9A-Fa-f]{1,4}){1,6}|:((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)|:))|(:((:[0-9A-Fa-f]{1,4}){1,7}|:)))(\/([0-9]|[1-9][0-9]|1[0-1][0-9]|12[0-8]))?$ regexFailureMessage: You must enter a valid IP address or CIDR. helpLink: suricata.html - EXTERNAL_NET: - description: List of hosts or networks. - helpLink: suricata.html - HTTP_SERVERS: - description: List of hosts or networks. - helpLink: suricata.html - SMTP_SERVERS: - description: List of hosts or networks. - helpLink: suricata.html - SQL_SERVERS: - description: List of hosts or networks. - helpLink: suricata.html - DNS_SERVERS: - description: List of hosts or networks. - helpLink: suricata.html - TELNET_SERVERS: - description: List of hosts or networks. - helpLink: suricata.html - AIM_SERVERS: - description: List of hosts or networks. - helpLink: suricata.html - DC_SERVERS: - description: List of hosts or networks. - helpLink: suricata.html - DNP3_SERVER: - description: List of hosts or networks. - helpLink: suricata.html - DNP3_CLIENT: - description: List of hosts or networks. - helpLink: suricata.html - MODBUS_CLIENT: - description: List of hosts or networks. - helpLink: suricata.html - MODBUS_SERVER: - description: List of hosts or networks. - helpLink: suricata.html - ENIP_CLIENT: - description: List of hosts or networks. - helpLink: suricata.html - ENIP_SERVER: - description: List of hosts or networks. - helpLink: suricata.html + duplicates: True + EXTERNAL_NET: *suriaddressgroup + HTTP_SERVERS: *suriaddressgroup + SMTP_SERVERS: *suriaddressgroup + SQL_SERVERS: *suriaddressgroup + DNS_SERVERS: *suriaddressgroup + TELNET_SERVERS: *suriaddressgroup + AIM_SERVERS: *suriaddressgroup + DC_SERVERS: *suriaddressgroup + DNP3_SERVER: *suriaddressgroup + DNP3_CLIENT: *suriaddressgroup + MODBUS_CLIENT: *suriaddressgroup + MODBUS_SERVER: *suriaddressgroup + ENIP_CLIENT: *suriaddressgroup + ENIP_SERVER: *suriaddressgroup port-groups: - HTTP_PORTS: - description: List of ports to look for HTTP traffic on. - helpLink: suricata.html - SHELLCODE_PORTS: - description: List of ports to look for SHELLCODE traffic on. - helpLink: suricata.html - ORACLE_PORTS: - description: List of ports to look for ORACLE traffic on. - helpLink: suricata.html - SSH_PORTS: - description: List of ports to look for SSH traffic on. - helpLink: suricata.html - DNP3_PORTS: - description: List of ports to look for DNP3 traffic on. - helpLink: suricata.html - MODBUS_PORTS: - description: List of ports to look for MODBUS traffic on. - helpLink: suricata.html - FILE_DATA_PORTS: - description: List of ports to look for FILE_DATA traffic on. - helpLink: suricata.html - FTP_PORTS: - description: List of ports to look for FTP traffic on. - helpLink: suricata.html - VXLAN_PORTS: - description: List of ports to look for VXLAN traffic on. - helpLink: suricata.html - TEREDO_PORTS: - description: List of ports to look for TEREDO traffic on. + HTTP_PORTS: &suriportgroup + description: Assign a list of network port numbers to this Suricata variable. The variable can then be re-used within Suricata rules. This allows for a single adjustment to the variable that will then affect all rules referencing the variable. helpLink: suricata.html + duplicates: True + SHELLCODE_PORTS: *suriportgroup + ORACLE_PORTS: *suriportgroup + SSH_PORTS: *suriportgroup + DNP3_PORTS: *suriportgroup + MODBUS_PORTS: *suriportgroup + FILE_DATA_PORTS: *suriportgroup + FTP_PORTS: *suriportgroup + VXLAN_PORTS: *suriportgroup + TEREDO_PORTS: *suriportgroup outputs: eve-log: types: diff --git a/salt/zeek/soc_zeek.yaml b/salt/zeek/soc_zeek.yaml index bd5d88116..021bf29ea 100644 --- a/salt/zeek/soc_zeek.yaml +++ b/salt/zeek/soc_zeek.yaml @@ -19,13 +19,14 @@ zeek: helpLink: zeek.html networks: HOME_NET: - description: List of IP or CIDR blocks to define as the HOME_NET. + description: List of IP or CIDR blocks to define as the for this Zeek network alias. forcedType: "[]string" advanced: False helpLink: zeek.html multiline: True regex: ^(((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\/([0-9]|[1-2][0-9]|3[0-2]))?$|^((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?))|:))|(([0-9A-Fa-f]{1,4}:){5}((:[0-9A-Fa-f]{1,4}){1,2}|:((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)|:))|(([0-9A-Fa-f]{1,4}:){4}((:[0-9A-Fa-f]{1,4}){1,3}|:((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)|:))|(([0-9A-Fa-f]{1,4}:){3}((:[0-9A-Fa-f]{1,4}){1,4}|:((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)|:))|(([0-9A-Fa-f]{1,4}:){2}((:[0-9A-Fa-f]{1,4}){1,5}|:((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)|:))|(([0-9A-Fa-f]{1,4}:){1}((:[0-9A-Fa-f]{1,4}){1,6}|:((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)|:))|(:((:[0-9A-Fa-f]{1,4}){1,7}|:)))(\/([0-9]|[1-9][0-9]|1[0-1][0-9]|12[0-8]))?$ regexFailureMessage: You must enter a valid IP address or CIDR. + duplicates: True node: lb_procs: description: Contains the number of CPU cores or workers used by Zeek. This setting should only be applied to individual nodes and will be ignored if CPU affinity is enabled. @@ -60,6 +61,7 @@ zeek: file: True global: True advanced: True + duplicates: True file_extraction: description: Contains a list of file or MIME types Zeek will extract from the network streams. Values must adhere to the following format - {"MIME_TYPE":"FILE_EXTENSION"} helpLink: zeek.html From 9a4a85e3aed340bdcab22ce54b6a4cd6805f3788 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Wed, 1 May 2024 07:54:38 -0400 Subject: [PATCH 48/63] FEATURE: Lower EVAL memory requirement to 8GB RAM #12896 --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 60908e0d4..7afc0a883 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -559,7 +559,7 @@ check_requirements() { local num_nics=${#nic_list[@]} if [[ $is_eval ]]; then - req_mem=12 + req_mem=8 req_cores=4 req_nics=2 elif [[ $is_standalone ]]; then From 87c6d0a820f97eb9a05b5469cbce1a2c0f2c9ca7 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 1 May 2024 09:29:36 -0400 Subject: [PATCH 49/63] zeek networks will only ever have one HOME_NETWORKS setting --- salt/zeek/soc_zeek.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/zeek/soc_zeek.yaml b/salt/zeek/soc_zeek.yaml index 021bf29ea..ea2c948ba 100644 --- a/salt/zeek/soc_zeek.yaml +++ b/salt/zeek/soc_zeek.yaml @@ -26,7 +26,6 @@ zeek: multiline: True regex: ^(((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\/([0-9]|[1-2][0-9]|3[0-2]))?$|^((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?))|:))|(([0-9A-Fa-f]{1,4}:){5}((:[0-9A-Fa-f]{1,4}){1,2}|:((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)|:))|(([0-9A-Fa-f]{1,4}:){4}((:[0-9A-Fa-f]{1,4}){1,3}|:((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)|:))|(([0-9A-Fa-f]{1,4}:){3}((:[0-9A-Fa-f]{1,4}){1,4}|:((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)|:))|(([0-9A-Fa-f]{1,4}:){2}((:[0-9A-Fa-f]{1,4}){1,5}|:((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)|:))|(([0-9A-Fa-f]{1,4}:){1}((:[0-9A-Fa-f]{1,4}){1,6}|:((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)|:))|(:((:[0-9A-Fa-f]{1,4}){1,7}|:)))(\/([0-9]|[1-9][0-9]|1[0-1][0-9]|12[0-8]))?$ regexFailureMessage: You must enter a valid IP address or CIDR. - duplicates: True node: lb_procs: description: Contains the number of CPU cores or workers used by Zeek. This setting should only be applied to individual nodes and will be ignored if CPU affinity is enabled. From d0e140cf7b2bd92bbcc6fb6bc2bab3f69ccdbded Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 1 May 2024 09:30:52 -0400 Subject: [PATCH 50/63] zeek networks will only ever have one HOME_NETWORKS setting --- salt/zeek/soc_zeek.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/zeek/soc_zeek.yaml b/salt/zeek/soc_zeek.yaml index ea2c948ba..9a41f3daf 100644 --- a/salt/zeek/soc_zeek.yaml +++ b/salt/zeek/soc_zeek.yaml @@ -19,7 +19,7 @@ zeek: helpLink: zeek.html networks: HOME_NET: - description: List of IP or CIDR blocks to define as the for this Zeek network alias. + description: List of IP or CIDR blocks to define as the HOME_NET forcedType: "[]string" advanced: False helpLink: zeek.html From 66563a4da0c6b85f767b2ba9184df2ef4952d206 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 1 May 2024 09:31:11 -0400 Subject: [PATCH 51/63] zeek networks will only ever have one HOME_NETWORKS setting --- salt/zeek/soc_zeek.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/zeek/soc_zeek.yaml b/salt/zeek/soc_zeek.yaml index 9a41f3daf..1594eed58 100644 --- a/salt/zeek/soc_zeek.yaml +++ b/salt/zeek/soc_zeek.yaml @@ -19,7 +19,7 @@ zeek: helpLink: zeek.html networks: HOME_NET: - description: List of IP or CIDR blocks to define as the HOME_NET + description: List of IP or CIDR blocks to define as the HOME_NET. forcedType: "[]string" advanced: False helpLink: zeek.html From 6294f751ee3a0b2cad295a6b9b2e0294a4460dfa Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 1 May 2024 10:59:41 -0400 Subject: [PATCH 52/63] Cold min_age to 60d --- salt/elasticsearch/defaults.yaml | 452 +++++++++++++++---------------- 1 file changed, 226 insertions(+), 226 deletions(-) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 53340340f..156483b03 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -152,7 +152,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -316,7 +316,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -432,7 +432,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -544,7 +544,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -649,7 +649,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -762,7 +762,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -870,7 +870,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -933,7 +933,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -977,7 +977,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -1021,7 +1021,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -1065,7 +1065,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -1109,7 +1109,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -1153,7 +1153,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -1197,7 +1197,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -1241,7 +1241,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -1285,7 +1285,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -1329,7 +1329,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -1373,7 +1373,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -1417,7 +1417,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -1461,7 +1461,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -1505,7 +1505,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -1549,7 +1549,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -1593,7 +1593,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -1637,7 +1637,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -1681,7 +1681,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -1725,7 +1725,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -1769,7 +1769,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -1813,7 +1813,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -1857,7 +1857,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -1901,7 +1901,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -1945,7 +1945,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -1989,7 +1989,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -2033,7 +2033,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -2077,7 +2077,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -2121,7 +2121,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -2165,7 +2165,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -2209,7 +2209,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -2253,7 +2253,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -2297,7 +2297,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -2341,7 +2341,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -2385,7 +2385,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -2429,7 +2429,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -2473,7 +2473,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -2517,7 +2517,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -2561,7 +2561,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -2605,7 +2605,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -2649,7 +2649,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -2693,7 +2693,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -2737,7 +2737,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -2781,7 +2781,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -2825,7 +2825,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -2869,7 +2869,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -2913,7 +2913,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -2957,7 +2957,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -3001,7 +3001,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -3045,7 +3045,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -3089,7 +3089,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -3133,7 +3133,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -3177,7 +3177,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -3221,7 +3221,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -3265,7 +3265,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -3309,7 +3309,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -3353,7 +3353,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -3397,7 +3397,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -3441,7 +3441,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -3485,7 +3485,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -3529,7 +3529,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -3573,7 +3573,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -3635,7 +3635,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -3696,7 +3696,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -3757,7 +3757,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -3815,7 +3815,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -3871,7 +3871,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -3927,7 +3927,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -3980,7 +3980,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -4038,7 +4038,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -4094,7 +4094,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -4150,7 +4150,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -4211,7 +4211,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -4267,7 +4267,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -4323,7 +4323,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -4379,7 +4379,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -4435,7 +4435,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -4491,7 +4491,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -4547,7 +4547,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -4603,7 +4603,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -4659,7 +4659,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -4715,7 +4715,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -4759,7 +4759,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -4803,7 +4803,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -4847,7 +4847,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -4891,7 +4891,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -4935,7 +4935,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -4979,7 +4979,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -5023,7 +5023,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -5067,7 +5067,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -5111,7 +5111,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -5155,7 +5155,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -5199,7 +5199,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -5243,7 +5243,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -5287,7 +5287,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -5331,7 +5331,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -5375,7 +5375,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -5419,7 +5419,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -5463,7 +5463,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -5507,7 +5507,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -5551,7 +5551,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -5595,7 +5595,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -5639,7 +5639,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -5683,7 +5683,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -5727,7 +5727,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -5771,7 +5771,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -5815,7 +5815,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -5859,7 +5859,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -5903,7 +5903,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -5947,7 +5947,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -5991,7 +5991,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -6035,7 +6035,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -6079,7 +6079,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -6123,7 +6123,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -6167,7 +6167,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -6211,7 +6211,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -6255,7 +6255,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -6299,7 +6299,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -6343,7 +6343,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -6387,7 +6387,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -6431,7 +6431,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -6475,7 +6475,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -6519,7 +6519,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -6563,7 +6563,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -6607,7 +6607,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -6651,7 +6651,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -6695,7 +6695,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -6739,7 +6739,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -6783,7 +6783,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -6827,7 +6827,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -6871,7 +6871,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -6915,7 +6915,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -6959,7 +6959,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -7003,7 +7003,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -7047,7 +7047,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -7091,7 +7091,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -7135,7 +7135,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -7179,7 +7179,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -7223,7 +7223,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -7267,7 +7267,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -7311,7 +7311,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -7355,7 +7355,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -7399,7 +7399,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -7443,7 +7443,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -7487,7 +7487,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -7531,7 +7531,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -7575,7 +7575,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -7619,7 +7619,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -7663,7 +7663,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -7741,7 +7741,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -7785,7 +7785,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -7829,7 +7829,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -7873,7 +7873,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -7917,7 +7917,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -7961,7 +7961,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -8005,7 +8005,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -8049,7 +8049,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -8093,7 +8093,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -8137,7 +8137,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -8181,7 +8181,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -8225,7 +8225,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -8269,7 +8269,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -8313,7 +8313,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -8357,7 +8357,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -8401,7 +8401,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -8445,7 +8445,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -8489,7 +8489,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -8533,7 +8533,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -8577,7 +8577,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -8621,7 +8621,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -8666,7 +8666,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -8711,7 +8711,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -8756,7 +8756,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -8801,7 +8801,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -8846,7 +8846,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -8890,7 +8890,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -8934,7 +8934,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -8978,7 +8978,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -9022,7 +9022,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -9066,7 +9066,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -9110,7 +9110,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -9154,7 +9154,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -9198,7 +9198,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -9242,7 +9242,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -9286,7 +9286,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -9330,7 +9330,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -9374,7 +9374,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -9418,7 +9418,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -9462,7 +9462,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -9506,7 +9506,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -9550,7 +9550,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -9594,7 +9594,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -9638,7 +9638,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -9682,7 +9682,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -9726,7 +9726,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -9770,7 +9770,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -9814,7 +9814,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -9858,7 +9858,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -9902,7 +9902,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -9946,7 +9946,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -9990,7 +9990,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -10034,7 +10034,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -10078,7 +10078,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -10122,7 +10122,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -10166,7 +10166,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -10210,7 +10210,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -10254,7 +10254,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -10298,7 +10298,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -10342,7 +10342,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -10386,7 +10386,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -10430,7 +10430,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -10474,7 +10474,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -10518,7 +10518,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -10565,7 +10565,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -10675,7 +10675,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -10785,7 +10785,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -10897,7 +10897,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -11008,7 +11008,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -11120,7 +11120,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} @@ -11233,7 +11233,7 @@ elasticsearch: actions: set_priority: priority: 0 - min_age: 30d + min_age: 60d delete: actions: delete: {} From fe2edeb2fb827cad279149974687dd2b36f0bc5d Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 1 May 2024 11:01:59 -0400 Subject: [PATCH 53/63] 30d to 60d --- salt/elasticsearch/soc_elasticsearch.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index 42262a178..cc92493fb 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -120,7 +120,7 @@ elasticsearch: helpLink: elasticsearch.html cold: min_age: - description: Minimum age of index. ex. 30d - This determines when the index should be moved to the cold tier. While still searchable, this tier is typically optimized for lower storage costs rather than search speed. + description: Minimum age of index. ex. 60d - This determines when the index should be moved to the cold tier. While still searchable, this tier is typically optimized for lower storage costs rather than search speed. regex: ^[0-9]{1,5}d$ forcedType: string global: True @@ -302,7 +302,7 @@ elasticsearch: helpLink: elasticsearch.html cold: min_age: - description: Minimum age of index. ex. 30d - This determines when the index should be moved to the cold tier. While still searchable, this tier is typically optimized for lower storage costs rather than search speed. + description: Minimum age of index. ex. 60d - This determines when the index should be moved to the cold tier. While still searchable, this tier is typically optimized for lower storage costs rather than search speed. regex: ^[0-9]{1,5}d$ forcedType: string global: True From c71af9127bdcb617a13dbc2a0d2718e6d3b24fbe Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 1 May 2024 11:47:38 -0400 Subject: [PATCH 54/63] mark detections settings as read-only via the UI --- salt/idstools/soc_idstools.yaml | 12 ++++++++---- salt/suricata/soc_suricata.yaml | 3 ++- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/salt/idstools/soc_idstools.yaml b/salt/idstools/soc_idstools.yaml index 0a2bf0cbf..f4389f8ba 100644 --- a/salt/idstools/soc_idstools.yaml +++ b/salt/idstools/soc_idstools.yaml @@ -19,33 +19,37 @@ idstools: helpLink: rules.html sids: disabled: - description: Contains the list of NIDS rules manually disabled across the grid. To disable a rule, add its Signature ID (SID) to the Current Grid Value box, one entry per line. To disable multiple rules, you can use regular expressions. + description: Contains the list of NIDS rules (or regex patterns) disabled across the grid. This setting is readonly; Use the Detections screen to disable rules. global: True multiline: True forcedType: "[]string" regex: \d*|re:.* helpLink: managing-alerts.html + readonlyUi: True enabled: - description: Contains the list of NIDS rules manually enabled across the grid. To enable a rule, add its Signature ID (SID) to the Current Grid Value box, one entry per line. To enable multiple rules, you can use regular expressions. + description: Contains the list of NIDS rules (or regex patterns) enabled across the grid. This setting is readonly; Use the Detections screen to enable rules. global: True multiline: True forcedType: "[]string" regex: \d*|re:.* helpLink: managing-alerts.html + readonlyUi: True modify: - description: Contains the list of NIDS rules that were modified from their default values. Entries must adhere to the following format - SID "REGEX_SEARCH_TERM" "REGEX_REPLACE_TERM" + description: Contains the list of NIDS rules (SID "REGEX_SEARCH_TERM" "REGEX_REPLACE_TERM"). This setting is readonly; Use the Detections screen to modify rules. global: True multiline: True forcedType: "[]string" helpLink: managing-alerts.html + readonlyUi: True rules: local__rules: - description: Contains the list of custom NIDS rules applied to the grid. To add custom NIDS rules to the grid, enter one rule per line in the Current Grid Value box. + description: Contains the list of custom NIDS rules applied to the grid. This setting is readonly; Use the Detections screen to adjust rules. file: True global: True advanced: True title: Local Rules helpLink: local-rules.html + readonlyUi: True filters__rules: description: If you are using Suricata for metadata, then you can set custom filters for that metadata here. file: True diff --git a/salt/suricata/soc_suricata.yaml b/salt/suricata/soc_suricata.yaml index 78c28f9e4..f7c3b2920 100644 --- a/salt/suricata/soc_suricata.yaml +++ b/salt/suricata/soc_suricata.yaml @@ -4,13 +4,14 @@ suricata: helpLink: suricata.html thresholding: sids__yaml: - description: Threshold SIDS List + description: Threshold SIDS List. This setting is readonly; Use the Detections screen to modify rules. syntax: yaml file: True global: True multiline: True title: SIDS helpLink: suricata.html + readonlyUi: True classification: classification__config: description: Classifications config file. From 8cd75902f229ecb0bc2dd63a98cf78ceb28bd68f Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 1 May 2024 11:47:51 -0400 Subject: [PATCH 55/63] Update config.sls --- salt/soc/config.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/soc/config.sls b/salt/soc/config.sls index af34f5e7c..18b292b27 100644 --- a/salt/soc/config.sls +++ b/salt/soc/config.sls @@ -82,7 +82,7 @@ socmotd: crondetectionsruntime: cron.present: - - name: /usr/local/bin/so-detections-runtime-status cron + - name: /usr/local/bin/so-detections-runtime-status cron > /opt/so/log/soc/detection-sync.log 2>&1 - identifier: detections-runtime-status - user: socore - minute: '*/10' From f7223f132ace0ea0b5d5ca22963a9a86543a1faf Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 1 May 2024 12:00:39 -0400 Subject: [PATCH 56/63] Update config.sls --- salt/soc/config.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/soc/config.sls b/salt/soc/config.sls index 18b292b27..545118dd9 100644 --- a/salt/soc/config.sls +++ b/salt/soc/config.sls @@ -84,7 +84,7 @@ crondetectionsruntime: cron.present: - name: /usr/local/bin/so-detections-runtime-status cron > /opt/so/log/soc/detection-sync.log 2>&1 - identifier: detections-runtime-status - - user: socore + - user: root - minute: '*/10' - hour: '*' - daymonth: '*' From 7122709bbf9dd99c70a555f3e0bae93ba6d252ba Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 1 May 2024 12:25:34 -0400 Subject: [PATCH 57/63] set Sigma rules based on role if defined and default if not --- salt/soc/defaults.yaml | 13 ++++++++++--- salt/soc/merged.map.jinja | 8 +++++--- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 9be17bcca..b66ae3bbe 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1257,9 +1257,16 @@ soc: allowRegex: '' autoUpdateEnabled: true autoEnabledSigmaRules: - - core+critical - - securityonion-resources+critical - - securityonion-resources+high + default: + - core+critical + - securityonion-resources+critical + - securityonion-resources+high + so-eval: + - securityonion-resources+critical + - securityonion-resources+high + so-import: + - securityonion-resources+critical + - securityonion-resources+high communityRulesImportFrequencySeconds: 28800 denyRegex: '' elastAlertRulesFolder: /opt/sensoroni/elastalert diff --git a/salt/soc/merged.map.jinja b/salt/soc/merged.map.jinja index ae68dc01f..222566dba 100644 --- a/salt/soc/merged.map.jinja +++ b/salt/soc/merged.map.jinja @@ -30,9 +30,11 @@ {# since cases is not a valid soc config item and only used for the map files, remove it from being placed in the config #} {% do SOCMERGED.config.server.modules.pop('cases') %} -{# do not automatically enable Sigma rules if install is Eval or Import #} -{% if grains['role'] in ['so-eval', 'so-import'] %} - {% do SOCMERGED.config.server.modules.elastalertengine.update({'autoEnabledSigmaRules': []}) %} +{# set Sigma rules based on role if defined and default if not #} +{% if GLOBALS.role in SOCMERGED.config.server.modules.elastalertengine.autoEnabledSigmaRules %} +{% do SOCMERGED.config.server.modules.elastalertengine.update({'autoEnabledSigmaRules': SOCMERGED.config.server.modules.elastalertengine.autoEnabledSigmaRules[GLOBALS.role]}) %} +{% else %} +{% do SOCMERGED.config.server.modules.elastalertengine.update({'autoEnabledSigmaRules': SOCMERGED.config.server.modules.elastalertengine.autoEnabledSigmaRules.default}) %} {% endif %} {# remove these modules if detections is disabled #} From 252d9a53203fa7be1a12edb21c6ab1d81dce4e72 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 1 May 2024 12:51:04 -0400 Subject: [PATCH 58/63] make rule settings advanced --- salt/idstools/soc_idstools.yaml | 3 +++ salt/suricata/soc_suricata.yaml | 1 + 2 files changed, 4 insertions(+) diff --git a/salt/idstools/soc_idstools.yaml b/salt/idstools/soc_idstools.yaml index f4389f8ba..ce8b56569 100644 --- a/salt/idstools/soc_idstools.yaml +++ b/salt/idstools/soc_idstools.yaml @@ -26,6 +26,7 @@ idstools: regex: \d*|re:.* helpLink: managing-alerts.html readonlyUi: True + advanced: true enabled: description: Contains the list of NIDS rules (or regex patterns) enabled across the grid. This setting is readonly; Use the Detections screen to enable rules. global: True @@ -34,6 +35,7 @@ idstools: regex: \d*|re:.* helpLink: managing-alerts.html readonlyUi: True + advanced: true modify: description: Contains the list of NIDS rules (SID "REGEX_SEARCH_TERM" "REGEX_REPLACE_TERM"). This setting is readonly; Use the Detections screen to modify rules. global: True @@ -41,6 +43,7 @@ idstools: forcedType: "[]string" helpLink: managing-alerts.html readonlyUi: True + advanced: true rules: local__rules: description: Contains the list of custom NIDS rules applied to the grid. This setting is readonly; Use the Detections screen to adjust rules. diff --git a/salt/suricata/soc_suricata.yaml b/salt/suricata/soc_suricata.yaml index f7c3b2920..75ad1e476 100644 --- a/salt/suricata/soc_suricata.yaml +++ b/salt/suricata/soc_suricata.yaml @@ -12,6 +12,7 @@ suricata: title: SIDS helpLink: suricata.html readonlyUi: True + advanced: true classification: classification__config: description: Classifications config file. From 10c8e4203c273b90dcfa2d76f48d3595520a6b18 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 1 May 2024 12:54:21 -0400 Subject: [PATCH 59/63] Update config.sls --- salt/soc/config.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/soc/config.sls b/salt/soc/config.sls index 545118dd9..b440b07fc 100644 --- a/salt/soc/config.sls +++ b/salt/soc/config.sls @@ -82,7 +82,7 @@ socmotd: crondetectionsruntime: cron.present: - - name: /usr/local/bin/so-detections-runtime-status cron > /opt/so/log/soc/detection-sync.log 2>&1 + - name: /usr/local/bin/so-detections-runtime-status cron - identifier: detections-runtime-status - user: root - minute: '*/10' From 47ba4c0f57b7f8a0ea756cb680b883fbc8d05317 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 1 May 2024 12:55:29 -0400 Subject: [PATCH 60/63] add new annotation for soc autoEnabledSigmaRules --- salt/soc/soc_soc.yaml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index 2001fb0c1..4b88a5f84 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -89,10 +89,13 @@ soc: advanced: True helpLink: sigma.html autoEnabledSigmaRules: - description: 'Sigma rules to automatically enable on initial import. Format is $Ruleset+$Level - for example, for the core community ruleset and critical level rules: core+critical' - global: True - advanced: True - helpLink: sigma.html + default: &autoEnabledSigmaRules + description: 'Sigma rules to automatically enable on initial import. Format is $Ruleset+$Level - for example, for the core community ruleset and critical level rules: core+critical. These will be applied based on role if defined and default if not.' + global: True + advanced: True + helpLink: sigma.html + so-eval: *autoEnabledSigmaRules + so-import: *autoEnabledSigmaRules denyRegex: description: 'Regex used to filter imported Sigma rules. Deny regex takes precedence over the Allow regex setting.' global: True From 3285ae93665990562ab58ada8ceeb09e98fd1c84 Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 1 May 2024 20:11:56 +0000 Subject: [PATCH 61/63] Update mappings for detection fields --- .../component/so/detection-mappings.json | 23 +++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/salt/elasticsearch/templates/component/so/detection-mappings.json b/salt/elasticsearch/templates/component/so/detection-mappings.json index 105a0ead2..5e51b872b 100644 --- a/salt/elasticsearch/templates/component/so/detection-mappings.json +++ b/salt/elasticsearch/templates/component/so/detection-mappings.json @@ -20,10 +20,12 @@ "so_detection": { "properties": { "publicId": { - "type": "text" + "ignore_above": 1024, + "type": "keyword" }, "title": { - "type": "text" + "ignore_above": 1024, + "type": "keyword" }, "severity": { "ignore_above": 1024, @@ -36,6 +38,18 @@ "description": { "type": "text" }, + "category": { + "ignore_above": 1024, + "type": "keyword" + }, + "product": { + "ignore_above": 1024, + "type": "keyword" + }, + "service": { + "ignore_above": 1024, + "type": "keyword" + }, "content": { "type": "text" }, @@ -49,7 +63,8 @@ "type": "boolean" }, "tags": { - "type": "text" + "ignore_above": 1024, + "type": "keyword" }, "ruleset": { "ignore_above": 1024, @@ -136,4 +151,4 @@ "_meta": { "ecs_version": "1.12.2" } -} \ No newline at end of file +} From 1be3e6204d2456717d73d65e096207c5d5a15873 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Thu, 2 May 2024 10:38:56 -0400 Subject: [PATCH 62/63] FIX: Improve File dashboard #12914 --- salt/soc/defaults.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index b75263fa1..506c85ba5 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1732,16 +1732,16 @@ soc: - name: Host Registry Changes description: Windows Registry changes query: 'event.category: registry | groupby event.action | groupby -sankey event.action host.name | groupby host.name | groupby event.dataset event.action | groupby process.executable | groupby registry.path | groupby process.executable registry.path' - - name: Host DNS & Process Mappings + - name: Host DNS and Process Mappings description: DNS queries mapped to originating processes query: 'event.category: network AND _exists_:process.executable AND (_exists_:dns.question.name OR _exists_:dns.answers.data) | groupby host.name | groupby -sankey host.name dns.question.name | groupby dns.question.name | groupby event.dataset event.type | groupby process.executable | groupby dns.answers.data' - name: Host Process Activity description: Process activity captured on an endpoint query: 'event.category:process | groupby host.name | groupby -sankey host.name user.name* | groupby user.name | groupby event.dataset event.action | groupby process.working_directory | groupby process.executable | groupby process.command_line | groupby process.parent.executable | groupby process.parent.command_line | groupby -sankey process.parent.executable process.executable | table soc_timestamp host.name user.name process.parent.name process.name event.action process.working_directory event.dataset' - - name: Host File Activity + - name: Host File and Process Mappings description: File activity captured on an endpoint - query: 'event.category: file AND _exists_:process.executable | groupby host.name | groupby -sankey host.name process.executable | groupby process.executable | groupby event.dataset event.action event.type | groupby file.name' - - name: Host Network & Process Mappings + query: 'event.category: file AND _exists_:process.name AND _exists_:process.executable | groupby host.name | groupby -sankey host.name process.name | groupby process.name | groupby process.executable | groupby event.dataset event.action event.type | groupby file.name' + - name: Host Network and Process Mappings description: Network activity mapped to originating processes query: 'event.category: network AND _exists_:process.executable | groupby event.action | groupby -sankey event.action host.name | groupby host.name | groupby -sankey host.name user.name | groupby user.name | groupby event.dataset* event.type* event.action* | groupby dns.question.name | groupby process.executable | groupby process.name | groupby source.ip | groupby destination.ip | groupby destination.port' - name: Sysmon Overview From 0822a46e94024e685fffcfb6e2eb693c94ec9097 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Thu, 2 May 2024 10:42:34 -0400 Subject: [PATCH 63/63] FIX: Improve File dashboard #12914 --- salt/soc/defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 506c85ba5..ad154e9d1 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1739,7 +1739,7 @@ soc: description: Process activity captured on an endpoint query: 'event.category:process | groupby host.name | groupby -sankey host.name user.name* | groupby user.name | groupby event.dataset event.action | groupby process.working_directory | groupby process.executable | groupby process.command_line | groupby process.parent.executable | groupby process.parent.command_line | groupby -sankey process.parent.executable process.executable | table soc_timestamp host.name user.name process.parent.name process.name event.action process.working_directory event.dataset' - name: Host File and Process Mappings - description: File activity captured on an endpoint + description: File activity mapped to originating processes query: 'event.category: file AND _exists_:process.name AND _exists_:process.executable | groupby host.name | groupby -sankey host.name process.name | groupby process.name | groupby process.executable | groupby event.dataset event.action event.type | groupby file.name' - name: Host Network and Process Mappings description: Network activity mapped to originating processes