From 9db6df0f14887eb3c3c6dbac7273444ad27b532b Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Sat, 4 Mar 2023 15:19:19 -0500 Subject: [PATCH 01/22] Initial updates for 2.4 fieldnames --- salt/soc/defaults.yaml | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 9a468902c..14e8182f3 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -7,19 +7,19 @@ soc: icon: fa-crosshairs target: links: - - '/#/hunt?q="{value|escape}" | groupby event.module event.dataset' + - '/#/hunt?q="{value|escape}" | groupby event.module* event.dataset' - name: actionCorrelate description: actionCorrelateHelp icon: fab fa-searchengin target: '' links: - - '/#/hunt?q=("{:log.id.fuid}" OR "{:log.id.uid}" OR "{:network.community_id}") | groupby event.module event.dataset' - - '/#/hunt?q=("{:log.id.fuid}" OR "{:log.id.uid}") | groupby event.module event.dataset' - - '/#/hunt?q=("{:log.id.fuid}" OR "{:network.community_id}") | groupby event.module event.dataset' - - '/#/hunt?q=("{:log.id.uid}" OR "{:network.community_id}") | groupby event.module event.dataset' - - '/#/hunt?q="{:log.id.fuid}" | groupby event.module event.dataset' - - '/#/hunt?q="{:log.id.uid}" | groupby event.module event.dataset' - - '/#/hunt?q="{:network.community_id}" | groupby event.module event.dataset' + - '/#/hunt?q=("{:log.id.fuid}" OR "{:log.id.uid}" OR "{:network.community_id}") | groupby event.module* event.dataset' + - '/#/hunt?q=("{:log.id.fuid}" OR "{:log.id.uid}") | groupby event.module* event.dataset' + - '/#/hunt?q=("{:log.id.fuid}" OR "{:network.community_id}") | groupby event.module* event.dataset' + - '/#/hunt?q=("{:log.id.uid}" OR "{:network.community_id}") | groupby event.module* event.dataset' + - '/#/hunt?q="{:log.id.fuid}" | groupby event.module* event.dataset' + - '/#/hunt?q="{:log.id.uid}" | groupby event.module* event.dataset' + - '/#/hunt?q="{:network.community_id}" | groupby event.module* event.dataset' - name: actionPcap description: actionPcapHelp icon: fa-stream @@ -560,7 +560,7 @@ soc: - destination.geo.country_iso_code - user.name - source.ip - ':sysmon:': + ':windows.sysmon_operational:': - soc_timestamp - event.dataset - process.executable @@ -1121,7 +1121,7 @@ soc: showSubtitle: true - name: Log Type description: Show all events grouped by module and dataset - query: '* | groupby event.module event.dataset' + query: '* | groupby event.module* event.dataset' showSubtitle: true - name: SOC Auth description: Users authenticated to SOC grouped by IP address and identity @@ -1145,11 +1145,11 @@ soc: showSubtitle: true - name: Sysmon Events description: Show all Sysmon logs grouped by event type - query: 'event.module:sysmon | groupby event.dataset' + query: 'event.dataset: windows.sysmon_operational | groupby event.action' showSubtitle: true - name: Sysmon Usernames description: Show all Sysmon logs grouped by username - query: 'event.module:sysmon | groupby event.dataset, user.name.keyword' + query: 'event.dataset: windows.sysmon_operational | groupby event.action, user.name.keyword' showSubtitle: true - name: Strelka description: Show all Strelka logs grouped by file type @@ -1380,7 +1380,7 @@ soc: queries: - name: Overview description: Overview of all events - query: '* | groupby -sankey event.dataset event.category* | groupby -pie event.category | groupby -bar event.module | groupby event.dataset | groupby event.module | groupby event.category | groupby observer.name | groupby source.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name' + query: '* | groupby -sankey event.dataset event.category* | groupby -pie event.category | groupby -bar event.module* | groupby event.dataset | groupby event.module* | groupby event.category | groupby observer.name | groupby source.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name' - name: SOC Auth description: SOC (Security Onion Console) authentication logs query: 'event.module:kratos AND event.dataset:audit AND msg:authenticated | groupby -sankey http_request.headers.x-real-ip identity_id | groupby http_request.headers.x-real-ip | groupby identity_id | groupby http_request.headers.user-agent' @@ -1389,28 +1389,28 @@ soc: query: '_index: "*:elastalert*" | groupby rule_name | groupby alert_info.type' - name: Alerts description: Overview of all alerts - query: 'event.dataset:alert | groupby event.module | groupby rule.name | groupby event.severity | groupby source.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name' + query: 'event.dataset:alert | groupby event.module* | groupby rule.name | groupby event.severity | groupby source.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name' - name: NIDS Alerts description: NIDS (Network Intrusion Detection System) alerts query: 'event.category:network AND event.dataset:alert | groupby rule.category | groupby -sankey source.ip destination.ip | groupby rule.name | groupby rule.uuid | groupby rule.gid | groupby source.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name' - name: Sysmon Overview description: Overview of all Sysmon data types - query: 'event.module:sysmon | groupby -sankey event.dataset winlog.computer_name | groupby -sankey winlog.computer_name user.name | groupby winlog.computer_name | groupby event.dataset | groupby user.name | groupby dns.query.name | groupby process.executable | groupby winlog.event_data.TargetObject | groupby file.target | groupby source.ip | groupby destination.ip | groupby destination.port' + query: 'event.dataset:windows.sysmon_operational | groupby -sankey event.action host.name | groupby -sankey host.name user.name | groupby host.name | groupby event.action | groupby user.name | groupby dns.query.name | groupby process.executable | groupby winlog.event_data.TargetObject | groupby file.target | groupby source.ip | groupby destination.ip | groupby destination.port' - name: Sysmon Registry description: Registry changes captured by Sysmon - query: '(event.dataset:registry_create_delete OR event.dataset:registry_value_set OR event.dataset:registry_key_value_rename) | groupby -sankey event.dataset winlog.computer_name | groupby winlog.computer_name | groupby event.dataset | groupby process.executable | groupby winlog.event_data.TargetObject | groupby process.executable winlog.event_data.TargetObject' + query: '(event.dataset:windows.sysmon_operational AND event.action:Registry*) | groupby -sankey event.action host.name | groupby host.name | groupby event.action | groupby process.executable | groupby registry.path | groupby process.executable registry.path' - name: Sysmon DNS description: DNS queries captured by Sysmon - query: 'event.dataset:dns_query | groupby -sankey winlog.computer_name dns.query.name | groupby winlog.computer_name | groupby process.executable | groupby dns.query.name | groupby dns.answers.name' + query: 'event.dataset:windows.sysmon_operational AND event.action:"Dns query (rule: DnsQuery)" | groupby -sankey host.name dns.query.name | groupby host.name | groupby process.executable | groupby dns.query.name | groupby dns.answers.name' - name: Sysmon Process description: Process activity captured by Sysmon - query: '(event.dataset:process_creation OR event.dataset:process_terminated OR event.dataset:process_access) | groupby -sankey winlog.computer_name user.name | groupby winlog.computer_name | groupby user.name | groupby event.dataset | groupby process.working_directory | groupby process.executable | groupby process.command_line | groupby process.parent.executable | groupby process.parent.command_line | groupby -sankey process.parent.executable process.executable' + query: '(event.dataset:process_creation OR event.dataset:process_terminated OR event.dataset:process_access) | groupby -sankey host.name user.name | groupby host.name | groupby user.name | groupby event.dataset | groupby process.working_directory | groupby process.executable | groupby process.command_line | groupby process.parent.executable | groupby process.parent.command_line | groupby -sankey process.parent.executable process.executable' - name: Sysmon File description: File activity captured by Sysmon - query: 'event.module:sysmon AND event.dataset:file_* | groupby -sankey winlog.computer_name process.executable | groupby winlog.computer_name | groupby event.dataset | groupby file.target | groupby process.executable' + query: 'event.module:sysmon AND event.dataset:file_* | groupby -sankey host.name process.executable | groupby host.name | groupby event.dataset | groupby file.target | groupby process.executable' - name: Sysmon Network description: Network activity captured by Sysmon - query: 'event.dataset:network_connection | groupby -sankey winlog.computer_name destination.ip destination.port | groupby winlog.computer_name | groupby user.name | groupby process.executable | groupby source.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name' + query: 'event.dataset:network_connection | groupby -sankey host.name destination.ip destination.port | groupby winlog.computer_name | groupby user.name | groupby process.executable | groupby source.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name' - name: Strelka description: Strelka file analysis query: 'event.module:strelka | groupby file.mime_type | groupby -sankey file.mime_type file.source | groupby file.source | groupby file.name' @@ -1611,7 +1611,7 @@ soc: - acknowledged queries: - name: 'Group By Name, Module' - query: '* | groupby rule.name event.module event.severity_label' + query: '* | groupby rule.name event.module* event.severity_label' - name: 'Group By Sensor, Source IP/Port, Destination IP/Port, Name' query: '* | groupby observer.name source.ip source.port destination.ip destination.port rule.name network.community_id event.severity_label' - name: 'Group By Source IP, Name' From a5c89bfaa16688c0cb46f155d843595d67bc5a97 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 8 Mar 2023 16:49:34 -0500 Subject: [PATCH 02/22] update sysmon dashboards --- salt/soc/defaults.yaml | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 14e8182f3..1b455c62a 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1395,22 +1395,22 @@ soc: query: 'event.category:network AND event.dataset:alert | groupby rule.category | groupby -sankey source.ip destination.ip | groupby rule.name | groupby rule.uuid | groupby rule.gid | groupby source.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name' - name: Sysmon Overview description: Overview of all Sysmon data types - query: 'event.dataset:windows.sysmon_operational | groupby -sankey event.action host.name | groupby -sankey host.name user.name | groupby host.name | groupby event.action | groupby user.name | groupby dns.query.name | groupby process.executable | groupby winlog.event_data.TargetObject | groupby file.target | groupby source.ip | groupby destination.ip | groupby destination.port' - - name: Sysmon Registry - description: Registry changes captured by Sysmon - query: '(event.dataset:windows.sysmon_operational AND event.action:Registry*) | groupby -sankey event.action host.name | groupby host.name | groupby event.action | groupby process.executable | groupby registry.path | groupby process.executable registry.path' - - name: Sysmon DNS - description: DNS queries captured by Sysmon - query: 'event.dataset:windows.sysmon_operational AND event.action:"Dns query (rule: DnsQuery)" | groupby -sankey host.name dns.query.name | groupby host.name | groupby process.executable | groupby dns.query.name | groupby dns.answers.name' - - name: Sysmon Process - description: Process activity captured by Sysmon - query: '(event.dataset:process_creation OR event.dataset:process_terminated OR event.dataset:process_access) | groupby -sankey host.name user.name | groupby host.name | groupby user.name | groupby event.dataset | groupby process.working_directory | groupby process.executable | groupby process.command_line | groupby process.parent.executable | groupby process.parent.command_line | groupby -sankey process.parent.executable process.executable' - - name: Sysmon File - description: File activity captured by Sysmon - query: 'event.module:sysmon AND event.dataset:file_* | groupby -sankey host.name process.executable | groupby host.name | groupby event.dataset | groupby file.target | groupby process.executable' - - name: Sysmon Network - description: Network activity captured by Sysmon - query: 'event.dataset:network_connection | groupby -sankey host.name destination.ip destination.port | groupby winlog.computer_name | groupby user.name | groupby process.executable | groupby source.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name' + query: 'event.dataset:windows.sysmon_operational | groupby -sankey event.action host.name | groupby -sankey host.name user.name | groupby host.name | groupby event.category event.action | groupby user.name | groupby dns.question.name | groupby process.executable | groupby winlog.event_data.TargetObject | groupby file.name | groupby source.ip | groupby destination.ip | groupby destination.port' + - name: Host Data - Registry Changes + description: Windows Registry changes + query: 'event.category: registry | groupby -sankey event.action host.name | groupby event.dataset event.action | groupby host.name | groupby process.executable | groupby registry.path | groupby process.executable registry.path' + - name: Host Data - DNS & Process Mappings + description: DNS queries mapped to originating processes + query: 'event.category: network AND _exists_:process.executable AND (_exists_:dns.question.name OR _exists_:dns.answers.data) | groupby -sankey host.name dns.question.name | groupby event.provider event.type | groupby host.name | groupby process.executable | groupby dns.question.name | groupby dns.answers.data' + - name: Host Data - Process + description: Process activity captured on an endpoint + query: 'event.category:process | groupby -sankey host.name user.name* | groupby event.dataset event.action | groupby host.name | groupby user.name | groupby process.working_directory | groupby process.executable | groupby process.command_line | groupby process.parent.executable | groupby process.parent.command_line | groupby -sankey process.parent.executable process.executable' + - name: Host Data - File + description: File activity captured on an endpoint + query: 'event.category: file AND _exists_:process.executable | groupby -sankey host.name process.executable | groupby host.name | groupby event.provider event.action event.type | groupby file.name | groupby process.executable' + - name: Host Data - Network & Process Mappings + description: Network activity mapped to originating processes + query: 'event.category: network AND _exists_:process.executable | groupby -sankey event.action host.name | groupby -sankey host.name user.name | groupby event.provider* event.type* event.action* | groupby host.name | groupby user.name | groupby dns.question.name | groupby process.executable | groupby winlog.event_data.TargetObject | groupby process.name | groupby source.ip | groupby destination.ip | groupby destination.port' - name: Strelka description: Strelka file analysis query: 'event.module:strelka | groupby file.mime_type | groupby -sankey file.mime_type file.source | groupby file.source | groupby file.name' @@ -1432,9 +1432,11 @@ soc: - name: DPD description: DPD (Dynamic Protocol Detection) errors query: 'event.dataset:dpd | groupby error.reason | groupby network.protocol | groupby -sankey source.ip destination.ip | groupby source.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name' + - name: Files description: Files seen in network traffic query: 'event.dataset:file | groupby file.mime_type | groupby -sankey file.mime_type file.source | groupby file.source | groupby file.bytes.total | groupby source.ip | groupby destination.ip | groupby destination_geo.organization_name' + - name: FTP description: FTP (File Transfer Protocol) network metadata query: 'event.dataset:ftp | groupby -sankey ftp.command destination.ip | groupby ftp.command | groupby ftp.argument | groupby ftp.user | groupby source.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name' From 14938060406185b9177c601699323047db0b6869 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 8 Mar 2023 17:03:02 -0500 Subject: [PATCH 03/22] Change host dashboard titles --- salt/soc/defaults.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 1b455c62a..aa78ce3e2 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1396,19 +1396,19 @@ soc: - name: Sysmon Overview description: Overview of all Sysmon data types query: 'event.dataset:windows.sysmon_operational | groupby -sankey event.action host.name | groupby -sankey host.name user.name | groupby host.name | groupby event.category event.action | groupby user.name | groupby dns.question.name | groupby process.executable | groupby winlog.event_data.TargetObject | groupby file.name | groupby source.ip | groupby destination.ip | groupby destination.port' - - name: Host Data - Registry Changes + - name: Host Registry Changes description: Windows Registry changes query: 'event.category: registry | groupby -sankey event.action host.name | groupby event.dataset event.action | groupby host.name | groupby process.executable | groupby registry.path | groupby process.executable registry.path' - - name: Host Data - DNS & Process Mappings + - name: Host DNS & Process Mappings description: DNS queries mapped to originating processes query: 'event.category: network AND _exists_:process.executable AND (_exists_:dns.question.name OR _exists_:dns.answers.data) | groupby -sankey host.name dns.question.name | groupby event.provider event.type | groupby host.name | groupby process.executable | groupby dns.question.name | groupby dns.answers.data' - - name: Host Data - Process + - name: Host Process Activity description: Process activity captured on an endpoint query: 'event.category:process | groupby -sankey host.name user.name* | groupby event.dataset event.action | groupby host.name | groupby user.name | groupby process.working_directory | groupby process.executable | groupby process.command_line | groupby process.parent.executable | groupby process.parent.command_line | groupby -sankey process.parent.executable process.executable' - - name: Host Data - File + - name: Host File Activity description: File activity captured on an endpoint query: 'event.category: file AND _exists_:process.executable | groupby -sankey host.name process.executable | groupby host.name | groupby event.provider event.action event.type | groupby file.name | groupby process.executable' - - name: Host Data - Network & Process Mappings + - name: Host Network & Process Mappings description: Network activity mapped to originating processes query: 'event.category: network AND _exists_:process.executable | groupby -sankey event.action host.name | groupby -sankey host.name user.name | groupby event.provider* event.type* event.action* | groupby host.name | groupby user.name | groupby dns.question.name | groupby process.executable | groupby winlog.event_data.TargetObject | groupby process.name | groupby source.ip | groupby destination.ip | groupby destination.port' - name: Strelka From 73abf8dbfdc0dd57ef8e03798f168b102ff7a78c Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Thu, 9 Mar 2023 14:32:52 -0500 Subject: [PATCH 04/22] Generic host dashboard --- salt/soc/defaults.yaml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index aa78ce3e2..e516631fe 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -562,11 +562,11 @@ soc: - source.ip ':windows.sysmon_operational:': - soc_timestamp - - event.dataset + - event.action - process.executable - user.name - file.target - - dns.query.name + - dns.question.name - winlog.event_data.TargetObject '::network_connection': - soc_timestamp @@ -1116,7 +1116,7 @@ soc: enabled: true queries: - name: Default Query - description: Show all events grouped by the origin host + description: Show all events grouped by the observer host query: '* | groupby observer.name' showSubtitle: true - name: Log Type @@ -1396,21 +1396,24 @@ soc: - name: Sysmon Overview description: Overview of all Sysmon data types query: 'event.dataset:windows.sysmon_operational | groupby -sankey event.action host.name | groupby -sankey host.name user.name | groupby host.name | groupby event.category event.action | groupby user.name | groupby dns.question.name | groupby process.executable | groupby winlog.event_data.TargetObject | groupby file.name | groupby source.ip | groupby destination.ip | groupby destination.port' + - name: Host Overview + description: Overview of all host data types + query: '((event.category:registry OR event.category:host OR event.category:process OR event.category:driver OR event.category:configuration) OR (event.category:file AND _exists_:process.executable) OR (event.category:network AND _exists_:host.name)) | groupby event.dataset* event.category* event.action* | groupby event.type | groupby host.name | groupby user.name | groupby file.name | groupby process.executable' - name: Host Registry Changes description: Windows Registry changes query: 'event.category: registry | groupby -sankey event.action host.name | groupby event.dataset event.action | groupby host.name | groupby process.executable | groupby registry.path | groupby process.executable registry.path' - name: Host DNS & Process Mappings description: DNS queries mapped to originating processes - query: 'event.category: network AND _exists_:process.executable AND (_exists_:dns.question.name OR _exists_:dns.answers.data) | groupby -sankey host.name dns.question.name | groupby event.provider event.type | groupby host.name | groupby process.executable | groupby dns.question.name | groupby dns.answers.data' + query: 'event.category: network AND _exists_:process.executable AND (_exists_:dns.question.name OR _exists_:dns.answers.data) | groupby -sankey host.name dns.question.name | groupby event.dataset event.type | groupby host.name | groupby process.executable | groupby dns.question.name | groupby dns.answers.data' - name: Host Process Activity description: Process activity captured on an endpoint query: 'event.category:process | groupby -sankey host.name user.name* | groupby event.dataset event.action | groupby host.name | groupby user.name | groupby process.working_directory | groupby process.executable | groupby process.command_line | groupby process.parent.executable | groupby process.parent.command_line | groupby -sankey process.parent.executable process.executable' - name: Host File Activity description: File activity captured on an endpoint - query: 'event.category: file AND _exists_:process.executable | groupby -sankey host.name process.executable | groupby host.name | groupby event.provider event.action event.type | groupby file.name | groupby process.executable' + query: 'event.category: file AND _exists_:process.executable | groupby -sankey host.name process.executable | groupby host.name | groupby event.dataset event.action event.type | groupby file.name | groupby process.executable' - name: Host Network & Process Mappings description: Network activity mapped to originating processes - query: 'event.category: network AND _exists_:process.executable | groupby -sankey event.action host.name | groupby -sankey host.name user.name | groupby event.provider* event.type* event.action* | groupby host.name | groupby user.name | groupby dns.question.name | groupby process.executable | groupby winlog.event_data.TargetObject | groupby process.name | groupby source.ip | groupby destination.ip | groupby destination.port' + query: 'event.category: network AND _exists_:process.executable | groupby -sankey event.action host.name | groupby -sankey host.name user.name | groupby event.dataset* event.type* event.action* | groupby host.name | groupby user.name | groupby dns.question.name | groupby process.executable | groupby winlog.event_data.TargetObject | groupby process.name | groupby source.ip | groupby destination.ip | groupby destination.port' - name: Strelka description: Strelka file analysis query: 'event.module:strelka | groupby file.mime_type | groupby -sankey file.mime_type file.source | groupby file.source | groupby file.name' @@ -1432,11 +1435,9 @@ soc: - name: DPD description: DPD (Dynamic Protocol Detection) errors query: 'event.dataset:dpd | groupby error.reason | groupby network.protocol | groupby -sankey source.ip destination.ip | groupby source.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name' - - name: Files description: Files seen in network traffic query: 'event.dataset:file | groupby file.mime_type | groupby -sankey file.mime_type file.source | groupby file.source | groupby file.bytes.total | groupby source.ip | groupby destination.ip | groupby destination_geo.organization_name' - - name: FTP description: FTP (File Transfer Protocol) network metadata query: 'event.dataset:ftp | groupby -sankey ftp.command destination.ip | groupby ftp.command | groupby ftp.argument | groupby ftp.user | groupby source.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name' From 16d94781967a96caf3afda24783ce6ee2c38ff5d Mon Sep 17 00:00:00 2001 From: weslambert Date: Fri, 10 Mar 2023 16:54:47 -0500 Subject: [PATCH 05/22] Add index lifecycle management policy definitions for default Elastic Agent data streams --- salt/elasticsearch/defaults.yaml | 209 +++++++++++++++++++++++++++++++ 1 file changed, 209 insertions(+) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index a0c431881..c4098e08c 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -84,6 +84,25 @@ elasticsearch: - "so-fleet_globals-1" - "so-fleet_agent_id_verification-1" priority: 200 + policy: + phases: + hot: + min_age: 0ms + actions: + set_priority: + priority: 100 + rollover: + max_age: 30d + max_primary_shard_size: 50gb + cold: + min_age: 30d + actions: + set_priority: + priority: 0 + delete: + min_age: 365d + actions: + delete: {} _meta: package: name: elastic_agent @@ -119,6 +138,25 @@ elasticsearch: - "so-fleet_globals-1" - "so-fleet_agent_id_verification-1" priority: 200 + policy: + phases: + hot: + min_age: 0ms + actions: + set_priority: + priority: 100 + rollover: + max_age: 30d + max_primary_shard_size: 50gb + cold: + min_age: 30d + actions: + set_priority: + priority: 0 + delete: + min_age: 365d + actions: + delete: {} _meta: package: name: elastic_agent @@ -154,6 +192,25 @@ elasticsearch: - "so-fleet_globals-1" - "so-fleet_agent_id_verification-1" priority: 200 + policy: + phases: + hot: + min_age: 0ms + actions: + set_priority: + priority: 100 + rollover: + max_age: 30d + max_primary_shard_size: 50gb + cold: + min_age: 30d + actions: + set_priority: + priority: 0 + delete: + min_age: 365d + actions: + delete: {} _meta: package: name: elastic_agent @@ -189,6 +246,25 @@ elasticsearch: - "so-fleet_globals-1" - "so-fleet_agent_id_verification-1" priority: 200 + policy: + phases: + hot: + min_age: 0ms + actions: + set_priority: + priority: 100 + rollover: + max_age: 30d + max_primary_shard_size: 50gb + cold: + min_age: 30d + actions: + set_priority: + priority: 0 + delete: + min_age: 365d + actions: + delete: {} _meta: package: name: elastic_agent @@ -224,6 +300,25 @@ elasticsearch: - "so-fleet_globals-1" - "so-fleet_agent_id_verification-1" priority: 200 + policy: + phases: + hot: + min_age: 0ms + actions: + set_priority: + priority: 100 + rollover: + max_age: 30d + max_primary_shard_size: 50gb + cold: + min_age: 30d + actions: + set_priority: + priority: 0 + delete: + min_age: 365d + actions: + delete: {} _meta: package: name: elastic_agent @@ -259,6 +354,25 @@ elasticsearch: - "so-fleet_globals-1" - "so-fleet_agent_id_verification-1" priority: 200 + policy: + phases: + hot: + min_age: 0ms + actions: + set_priority: + priority: 100 + rollover: + max_age: 30d + max_primary_shard_size: 50gb + cold: + min_age: 30d + actions: + set_priority: + priority: 0 + delete: + min_age: 365d + actions: + delete: {} _meta: package: name: elastic_agent @@ -294,6 +408,25 @@ elasticsearch: - "so-fleet_globals-1" - "so-fleet_agent_id_verification-1" priority: 200 + policy: + phases: + hot: + min_age: 0ms + actions: + set_priority: + priority: 100 + rollover: + max_age: 30d + max_primary_shard_size: 50gb + cold: + min_age: 30d + actions: + set_priority: + priority: 0 + delete: + min_age: 365d + actions: + delete: {} _meta: package: name: elastic_agent @@ -329,6 +462,25 @@ elasticsearch: - "so-fleet_globals-1" - "so-fleet_agent_id_verification-1" priority: 200 + policy: + phases: + hot: + min_age: 0ms + actions: + set_priority: + priority: 100 + rollover: + max_age: 30d + max_primary_shard_size: 50gb + cold: + min_age: 30d + actions: + set_priority: + priority: 0 + delete: + min_age: 365d + actions: + delete: {} _meta: package: name: elastic_agent @@ -364,6 +516,25 @@ elasticsearch: - "so-fleet_globals-1" - "so-fleet_agent_id_verification-1" priority: 200 + policy: + phases: + hot: + min_age: 0ms + actions: + set_priority: + priority: 100 + rollover: + max_age: 30d + max_primary_shard_size: 50gb + cold: + min_age: 30d + actions: + set_priority: + priority: 0 + delete: + min_age: 365d + actions: + delete: {} _meta: package: name: elastic_agent @@ -399,6 +570,25 @@ elasticsearch: - "so-fleet_globals-1" - "so-fleet_agent_id_verification-1" priority: 200 + policy: + phases: + hot: + min_age: 0ms + actions: + set_priority: + priority: 100 + rollover: + max_age: 30d + max_primary_shard_size: 50gb + cold: + min_age: 30d + actions: + set_priority: + priority: 0 + delete: + min_age: 365d + actions: + delete: {} _meta: package: name: elastic_agent @@ -434,6 +624,25 @@ elasticsearch: - "so-fleet_globals-1" - "so-fleet_agent_id_verification-1" priority: 200 + policy: + phases: + hot: + min_age: 0ms + actions: + set_priority: + priority: 100 + rollover: + max_age: 30d + max_primary_shard_size: 50gb + cold: + min_age: 30d + actions: + set_priority: + priority: 0 + delete: + min_age: 365d + actions: + delete: {} _meta: package: name: elastic_agent From e105e56facbb5c9639c1da1fb30b26ec27a14073 Mon Sep 17 00:00:00 2001 From: Wes Date: Mon, 13 Mar 2023 13:27:02 +0000 Subject: [PATCH 06/22] Move data stream configuration outside of ILM policy definition --- salt/elasticsearch/defaults.yaml | 57 ++++++++++++++------------------ 1 file changed, 24 insertions(+), 33 deletions(-) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index c4098e08c..d47125972 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -84,6 +84,9 @@ elasticsearch: - "so-fleet_globals-1" - "so-fleet_agent_id_verification-1" priority: 200 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: hot: @@ -108,9 +111,6 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - data_stream: - hidden: false - allow_custom_routing: false so-logs-elastic_agent.auditbeat: index_sorting: False index_template: @@ -138,6 +138,9 @@ elasticsearch: - "so-fleet_globals-1" - "so-fleet_agent_id_verification-1" priority: 200 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: hot: @@ -162,9 +165,6 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - data_stream: - hidden: false - allow_custom_routing: false so-logs-elastic_agent.cloudbeat: index_sorting: False index_template: @@ -216,9 +216,6 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - data_stream: - hidden: false - allow_custom_routing: false so-logs-elastic_agent.endpoint_security: index_sorting: False index_template: @@ -246,6 +243,9 @@ elasticsearch: - "so-fleet_globals-1" - "so-fleet_agent_id_verification-1" priority: 200 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: hot: @@ -270,9 +270,6 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - data_stream: - hidden: false - allow_custom_routing: false so-logs-elastic_agent.filebeat: index_sorting: False index_template: @@ -324,9 +321,6 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - data_stream: - hidden: false - allow_custom_routing: false so-logs-elastic_agent.fleet_server: index_sorting: False index_template: @@ -354,6 +348,9 @@ elasticsearch: - "so-fleet_globals-1" - "so-fleet_agent_id_verification-1" priority: 200 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: hot: @@ -378,9 +375,6 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - data_stream: - hidden: false - allow_custom_routing: false so-logs-elastic_agent.heartbeat: index_sorting: False index_template: @@ -432,9 +426,6 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - data_stream: - hidden: false - allow_custom_routing: false so-logs-elastic_agent: index_sorting: False index_template: @@ -462,6 +453,9 @@ elasticsearch: - "so-fleet_globals-1" - "so-fleet_agent_id_verification-1" priority: 200 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: hot: @@ -486,9 +480,6 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - data_stream: - hidden: false - allow_custom_routing: false so-logs-elastic_agent.metricbeat: index_sorting: False index_template: @@ -516,6 +507,9 @@ elasticsearch: - "so-fleet_globals-1" - "so-fleet_agent_id_verification-1" priority: 200 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: hot: @@ -540,9 +534,6 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - data_stream: - hidden: false - allow_custom_routing: false so-logs-elastic_agent.osquerybeat: index_sorting: False index_template: @@ -570,6 +561,9 @@ elasticsearch: - "so-fleet_globals-1" - "so-fleet_agent_id_verification-1" priority: 200 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: hot: @@ -594,9 +588,6 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - data_stream: - hidden: false - allow_custom_routing: false so-logs-elastic_agent.packetbeat: index_sorting: False index_template: @@ -624,6 +615,9 @@ elasticsearch: - "so-fleet_globals-1" - "so-fleet_agent_id_verification-1" priority: 200 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: hot: @@ -648,9 +642,6 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - data_stream: - hidden: false - allow_custom_routing: false so-aws: warm: 7 close: 30 From b3a2680847f4222caa290051859fbd716fea3f63 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Mon, 13 Mar 2023 11:41:36 -0400 Subject: [PATCH 07/22] auto-apply firewall rules --- salt/common/tools/sbin/so-firewall | 2 ++ salt/common/tools/sbin/so-firewall-minion | 12 ++++++------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/salt/common/tools/sbin/so-firewall b/salt/common/tools/sbin/so-firewall index 69808c709..16dcdf729 100755 --- a/salt/common/tools/sbin/so-firewall +++ b/salt/common/tools/sbin/so-firewall @@ -97,6 +97,8 @@ echo "$IP" >> $local_salt_dir/hostgroups/$ROLE if [ "$APPLY" = "true" ]; then echo "Applying the firewall rules" salt-call state.apply firewall queue=True + echo "Firewall rules have been applied... Review logs further if there were errors." + echo "" else echo "Firewall rules will be applied next salt run" fi diff --git a/salt/common/tools/sbin/so-firewall-minion b/salt/common/tools/sbin/so-firewall-minion index e796035f9..19ea26864 100755 --- a/salt/common/tools/sbin/so-firewall-minion +++ b/salt/common/tools/sbin/so-firewall-minion @@ -54,25 +54,25 @@ fi 'EVAL' | 'MANAGERSEARCH' | 'STANDALONE' | 'IMPORT') so-firewall --role=manager --ip="$IP" so-firewall --role=sensors --ip="$IP" - so-firewall --apply --role=searchnodes --ip="$IP" + so-firewall --apply=true --role=searchnodes --ip="$IP" ;; 'SENSOR' | 'SEARCHNODE' | 'HEAVYNODE' | 'IDH' | 'RECEIVER') case "$ROLE" in 'SENSOR') - so-firewall --apply --role=sensors --ip="$IP" + so-firewall --apply=true --role=sensors --ip="$IP" ;; 'SEARCHNODE') - so-firewall --apply --role=searchnodes --ip="$IP" + so-firewall --apply=true --role=searchnodes --ip="$IP" ;; 'HEAVYNODE') so-firewall --role=sensors --ip="$IP" - so-firewall --apply --role=heavynodes --ip="$IP" + so-firewall --apply=true --role=heavynodes --ip="$IP" ;; 'IDH') - so-firewall --apply --role=sensors --ip="$IP" + so-firewall --apply=true --role=sensors --ip="$IP" ;; 'RECEIVER') - so-firewall --apply --role=receivers --ip="$IP" + so-firewall --apply=true --role=receivers --ip="$IP" ;; esac ;; From f7be4ba31c48d7b808f9d31b4fa79c5ba09e5f61 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Mon, 13 Mar 2023 14:07:17 -0400 Subject: [PATCH 08/22] Remove host field from NIDS logs --- salt/elasticsearch/files/ingest/common.nids | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/files/ingest/common.nids b/salt/elasticsearch/files/ingest/common.nids index df6af7a85..53a3f7b79 100644 --- a/salt/elasticsearch/files/ingest/common.nids +++ b/salt/elasticsearch/files/ingest/common.nids @@ -11,7 +11,7 @@ { "set": { "if": "ctx.rule.severity == 3", "field": "event.severity", "value": 1, "override": true } }, { "set": { "if": "ctx.rule.severity == 2", "field": "event.severity", "value": 2, "override": true } }, { "set": { "if": "ctx.rule.severity == 1", "field": "event.severity", "value": 3, "override": true } }, - { "remove": { "field": ["rule_type", "rest_of_rulename"], "ignore_failure": true } }, + { "remove": { "field": ["rule_type", "rest_of_rulename", "host"], "ignore_failure": true } }, { "pipeline": { "name": "common" } } ] } From 8d395dc465911918c3a2633bffb58199b524e7fa Mon Sep 17 00:00:00 2001 From: Wes Date: Mon, 13 Mar 2023 20:54:13 +0000 Subject: [PATCH 09/22] Add Elastic Agent default data stream backing indices for management by Curator --- salt/curator/defaults.yaml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/salt/curator/defaults.yaml b/salt/curator/defaults.yaml index 237a50c81..958dd99ef 100644 --- a/salt/curator/defaults.yaml +++ b/salt/curator/defaults.yaml @@ -15,6 +15,27 @@ elasticsearch: logs-zeek-so: close: 30 delete: 365 + logs-elastic_agent-metricbeat-default: + close: 30 + delete: 365 + logs-elastic_agent-osquerybeat-default: + close: 30 + delete: 365 + logs-elastic_agent-fleet_server-default: + close: 30 + delete: 365 + logs-elastic_agent-filebeat-default: + close: 30 + delete: 365 + logs-elastic_agent-default: + close: 30 + delete: 365 + logs-system-auth-default: + close: 30 + delete: 365 + logs-system-syslog-default: + close: 30 + delete: 365 so-beats: close: 30 delete: 365 From efc58324999253c158915e710154206ffc671988 Mon Sep 17 00:00:00 2001 From: Wes Date: Mon, 13 Mar 2023 20:54:38 +0000 Subject: [PATCH 10/22] Add Elastic Agent default log action files --- .../logs-elastic_agent-default-close.yaml | 27 +++++++++++++++++++ .../logs-elastic_agent-default-delete.yaml | 27 +++++++++++++++++++ ...-elastic_agent-filebeat-default-close.yaml | 27 +++++++++++++++++++ ...elastic_agent-filebeat-default-delete.yaml | 27 +++++++++++++++++++ ...stic_agent-fleet_server-default-close.yaml | 27 +++++++++++++++++++ ...tic_agent-fleet_server-default-delete.yaml | 27 +++++++++++++++++++ ...lastic_agent-metricbeat-default-close.yaml | 27 +++++++++++++++++++ ...astic_agent-metricbeat-default-delete.yaml | 27 +++++++++++++++++++ ...astic_agent-osquerybeat-default-close.yaml | 27 +++++++++++++++++++ ...stic_agent-osquerybeat-default-delete.yaml | 27 +++++++++++++++++++ ...logs-elastic_agent-osquerybeat-delete.yaml | 27 +++++++++++++++++++ .../logs-system-auth-default-close.yaml | 27 +++++++++++++++++++ .../logs-system-auth-default-delete.yaml | 27 +++++++++++++++++++ .../action/logs-system-auth-syslog-close.yaml | 27 +++++++++++++++++++ .../logs-system-syslog-default-close.yaml | 27 +++++++++++++++++++ .../logs-system-syslog-default-delete.yaml | 27 +++++++++++++++++++ 16 files changed, 432 insertions(+) create mode 100644 salt/curator/files/action/logs-elastic_agent-default-close.yaml create mode 100644 salt/curator/files/action/logs-elastic_agent-default-delete.yaml create mode 100644 salt/curator/files/action/logs-elastic_agent-filebeat-default-close.yaml create mode 100644 salt/curator/files/action/logs-elastic_agent-filebeat-default-delete.yaml create mode 100644 salt/curator/files/action/logs-elastic_agent-fleet_server-default-close.yaml create mode 100644 salt/curator/files/action/logs-elastic_agent-fleet_server-default-delete.yaml create mode 100644 salt/curator/files/action/logs-elastic_agent-metricbeat-default-close.yaml create mode 100644 salt/curator/files/action/logs-elastic_agent-metricbeat-default-delete.yaml create mode 100644 salt/curator/files/action/logs-elastic_agent-osquerybeat-default-close.yaml create mode 100644 salt/curator/files/action/logs-elastic_agent-osquerybeat-default-delete.yaml create mode 100644 salt/curator/files/action/logs-elastic_agent-osquerybeat-delete.yaml create mode 100644 salt/curator/files/action/logs-system-auth-default-close.yaml create mode 100644 salt/curator/files/action/logs-system-auth-default-delete.yaml create mode 100644 salt/curator/files/action/logs-system-auth-syslog-close.yaml create mode 100644 salt/curator/files/action/logs-system-syslog-default-close.yaml create mode 100644 salt/curator/files/action/logs-system-syslog-default-delete.yaml diff --git a/salt/curator/files/action/logs-elastic_agent-default-close.yaml b/salt/curator/files/action/logs-elastic_agent-default-close.yaml new file mode 100644 index 000000000..ef03e4ba2 --- /dev/null +++ b/salt/curator/files/action/logs-elastic_agent-default-close.yaml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set cur_close_days = CURATORMERGED['logs-elastic_agent-default'].close %} +actions: + 1: + action: close + description: >- + Close Elastic Agent default indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-elastic_agent-default.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/logs-elastic_agent-default-delete.yaml b/salt/curator/files/action/logs-elastic_agent-default-delete.yaml new file mode 100644 index 000000000..dee51c758 --- /dev/null +++ b/salt/curator/files/action/logs-elastic_agent-default-delete.yaml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set DELETE_DAYS = CURATORMERGED['logs-elastic_agent-default'].delete %} +actions: + 1: + action: delete_indices + description: >- + Delete Elastic Agent default indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-elastic_agent-default.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + diff --git a/salt/curator/files/action/logs-elastic_agent-filebeat-default-close.yaml b/salt/curator/files/action/logs-elastic_agent-filebeat-default-close.yaml new file mode 100644 index 000000000..9277b25fd --- /dev/null +++ b/salt/curator/files/action/logs-elastic_agent-filebeat-default-close.yaml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set cur_close_days = CURATORMERGED['logs-elastic_agent.filebeat-default'].close %} +actions: + 1: + action: close + description: >- + Close Elastic Agent Filebeat indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-elastic_agent.filebeat-default.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/logs-elastic_agent-filebeat-default-delete.yaml b/salt/curator/files/action/logs-elastic_agent-filebeat-default-delete.yaml new file mode 100644 index 000000000..dfa51f260 --- /dev/null +++ b/salt/curator/files/action/logs-elastic_agent-filebeat-default-delete.yaml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set DELETE_DAYS = CURATORMERGED['logs-elastic_agent-filebeat-default'].delete %} +actions: + 1: + action: delete_indices + description: >- + Delete Elastic Agent Filebeat indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-elastic_agent.filebeat-default.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + diff --git a/salt/curator/files/action/logs-elastic_agent-fleet_server-default-close.yaml b/salt/curator/files/action/logs-elastic_agent-fleet_server-default-close.yaml new file mode 100644 index 000000000..6bc2026b9 --- /dev/null +++ b/salt/curator/files/action/logs-elastic_agent-fleet_server-default-close.yaml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set cur_close_days = CURATORMERGED['logs-elastic_agent-fleet_server-default'].close %} +actions: + 1: + action: close + description: >- + Close Elastic Agent Fleet Server indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-elastic_agent.fleet_server-default.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/logs-elastic_agent-fleet_server-default-delete.yaml b/salt/curator/files/action/logs-elastic_agent-fleet_server-default-delete.yaml new file mode 100644 index 000000000..6fa775ba8 --- /dev/null +++ b/salt/curator/files/action/logs-elastic_agent-fleet_server-default-delete.yaml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set DELETE_DAYS = CURATORMERGED['logs-elastic_agent-fleet_server-default'].delete %} +actions: + 1: + action: delete_indices + description: >- + Delete import indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-elastic_agent.fleet_server-default.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + diff --git a/salt/curator/files/action/logs-elastic_agent-metricbeat-default-close.yaml b/salt/curator/files/action/logs-elastic_agent-metricbeat-default-close.yaml new file mode 100644 index 000000000..a4e38cd8e --- /dev/null +++ b/salt/curator/files/action/logs-elastic_agent-metricbeat-default-close.yaml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set cur_close_days = CURATORMERGED['logs-elastic_agent-metricbeat-default'].close %} +actions: + 1: + action: close + description: >- + Close Elastic Agent Metricbeat indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-elastic_agent.metricbeat-default-.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/logs-elastic_agent-metricbeat-default-delete.yaml b/salt/curator/files/action/logs-elastic_agent-metricbeat-default-delete.yaml new file mode 100644 index 000000000..b42e42c83 --- /dev/null +++ b/salt/curator/files/action/logs-elastic_agent-metricbeat-default-delete.yaml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set DELETE_DAYS = CURATORMERGED['logs--elastic_agent-metricbeat-default'].delete %} +actions: + 1: + action: delete_indices + description: >- + Delete Elastic Agent Metricbeat indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-elastic_agent.metricbeat-default.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + diff --git a/salt/curator/files/action/logs-elastic_agent-osquerybeat-default-close.yaml b/salt/curator/files/action/logs-elastic_agent-osquerybeat-default-close.yaml new file mode 100644 index 000000000..9243d8cfb --- /dev/null +++ b/salt/curator/files/action/logs-elastic_agent-osquerybeat-default-close.yaml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set cur_close_days = CURATORMERGED['logs-elastic_agent-osquerybeat-default'].close %} +actions: + 1: + action: close + description: >- + Close Elastic Agent Osquerybeat indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-elastic_agent.osquerybeat-default.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/logs-elastic_agent-osquerybeat-default-delete.yaml b/salt/curator/files/action/logs-elastic_agent-osquerybeat-default-delete.yaml new file mode 100644 index 000000000..bce3b7e63 --- /dev/null +++ b/salt/curator/files/action/logs-elastic_agent-osquerybeat-default-delete.yaml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set DELETE_DAYS = CURATORMERGED['logs-elastic_agent-osquerybeat-default'].delete %} +actions: + 1: + action: delete_indices + description: >- + Delete Elastic Agent Osquerybeat indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-elastic_agent.osquerybeat-default.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + diff --git a/salt/curator/files/action/logs-elastic_agent-osquerybeat-delete.yaml b/salt/curator/files/action/logs-elastic_agent-osquerybeat-delete.yaml new file mode 100644 index 000000000..b46a5fc73 --- /dev/null +++ b/salt/curator/files/action/logs-elastic_agent-osquerybeat-delete.yaml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set DELETE_DAYS = CURATORMERGED['logs-import-so'].delete %} +actions: + 1: + action: delete_indices + description: >- + Delete import indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-import-so.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + diff --git a/salt/curator/files/action/logs-system-auth-default-close.yaml b/salt/curator/files/action/logs-system-auth-default-close.yaml new file mode 100644 index 000000000..7c04a0ca9 --- /dev/null +++ b/salt/curator/files/action/logs-system-auth-default-close.yaml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set cur_close_days = CURATORMERGED['logs-system.auth-default'].close %} +actions: + 1: + action: close + description: >- + Close Elastic Agent system auth indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-system.auth-default.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/logs-system-auth-default-delete.yaml b/salt/curator/files/action/logs-system-auth-default-delete.yaml new file mode 100644 index 000000000..d14d560f3 --- /dev/null +++ b/salt/curator/files/action/logs-system-auth-default-delete.yaml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set DELETE_DAYS = CURATORMERGED['logs-system.auth-default'].delete %} +actions: + 1: + action: delete_indices + description: >- + Delete Elastic Agent system auth indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-system.auth-default.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + diff --git a/salt/curator/files/action/logs-system-auth-syslog-close.yaml b/salt/curator/files/action/logs-system-auth-syslog-close.yaml new file mode 100644 index 000000000..52ddb5eb5 --- /dev/null +++ b/salt/curator/files/action/logs-system-auth-syslog-close.yaml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set cur_close_days = CURATORMERGED['logs-import-so'].close %} +actions: + 1: + action: close + description: >- + Close import indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-import-so.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/logs-system-syslog-default-close.yaml b/salt/curator/files/action/logs-system-syslog-default-close.yaml new file mode 100644 index 000000000..a9a697a66 --- /dev/null +++ b/salt/curator/files/action/logs-system-syslog-default-close.yaml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set cur_close_days = CURATORMERGED['logs-system.syslog-default'].close %} +actions: + 1: + action: close + description: >- + Close Elastic Agent system syslog indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-system.syslog-default.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/logs-system-syslog-default-delete.yaml b/salt/curator/files/action/logs-system-syslog-default-delete.yaml new file mode 100644 index 000000000..b46a5fc73 --- /dev/null +++ b/salt/curator/files/action/logs-system-syslog-default-delete.yaml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set DELETE_DAYS = CURATORMERGED['logs-import-so'].delete %} +actions: + 1: + action: delete_indices + description: >- + Delete import indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-import-so.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + From d5bb223235c6ac48cd69691a0b36419ea20cfb70 Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 13 Mar 2023 17:10:52 -0400 Subject: [PATCH 11/22] Fix system syslog delete file configuration --- .../files/action/logs-system-syslog-default-delete.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/curator/files/action/logs-system-syslog-default-delete.yaml b/salt/curator/files/action/logs-system-syslog-default-delete.yaml index b46a5fc73..36e079408 100644 --- a/salt/curator/files/action/logs-system-syslog-default-delete.yaml +++ b/salt/curator/files/action/logs-system-syslog-default-delete.yaml @@ -3,19 +3,19 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -{%- set DELETE_DAYS = CURATORMERGED['logs-import-so'].delete %} +{%- set DELETE_DAYS = CURATORMERGED['logs-system.syslog-default'].delete %} actions: 1: action: delete_indices description: >- - Delete import indices when older than {{ DELETE_DAYS }} days. + Delete Elastic Agent system syslog indices when older than {{ DELETE_DAYS }} days. options: ignore_empty_list: True disable_action: False filters: - filtertype: pattern kind: regex - value: '^(.ds-logs-import-so.*)$' + value: '^(.ds-logs-system.syslog-default.*)$' - filtertype: age source: name direction: older From c2701f1835372a75ed5ccb3fbca41561679fba3f Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 13 Mar 2023 17:24:12 -0400 Subject: [PATCH 12/22] Fix system syslog default key value --- .../curator/files/action/logs-system-syslog-default-delete.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/curator/files/action/logs-system-syslog-default-delete.yaml b/salt/curator/files/action/logs-system-syslog-default-delete.yaml index 36e079408..1a7d217e9 100644 --- a/salt/curator/files/action/logs-system-syslog-default-delete.yaml +++ b/salt/curator/files/action/logs-system-syslog-default-delete.yaml @@ -3,7 +3,7 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -{%- set DELETE_DAYS = CURATORMERGED['logs-system.syslog-default'].delete %} +{%- set DELETE_DAYS = CURATORMERGED['logs-system-syslog-default'].delete %} actions: 1: action: delete_indices From 8ade7b85fc450efbd9cb28ee5264b7ccd76213e7 Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 13 Mar 2023 17:24:40 -0400 Subject: [PATCH 13/22] Fix system syslog default key value --- salt/curator/files/action/logs-system-syslog-default-close.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/curator/files/action/logs-system-syslog-default-close.yaml b/salt/curator/files/action/logs-system-syslog-default-close.yaml index a9a697a66..3c9482b40 100644 --- a/salt/curator/files/action/logs-system-syslog-default-close.yaml +++ b/salt/curator/files/action/logs-system-syslog-default-close.yaml @@ -3,7 +3,7 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -{%- set cur_close_days = CURATORMERGED['logs-system.syslog-default'].close %} +{%- set cur_close_days = CURATORMERGED['logs-system-syslog-default'].close %} actions: 1: action: close From 785f100132bf6fc21010da55fad47450b1d8b666 Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 13 Mar 2023 17:25:33 -0400 Subject: [PATCH 14/22] Fix system auth default key value --- salt/curator/files/action/logs-system-auth-default-close.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/curator/files/action/logs-system-auth-default-close.yaml b/salt/curator/files/action/logs-system-auth-default-close.yaml index 7c04a0ca9..af9843b35 100644 --- a/salt/curator/files/action/logs-system-auth-default-close.yaml +++ b/salt/curator/files/action/logs-system-auth-default-close.yaml @@ -3,7 +3,7 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -{%- set cur_close_days = CURATORMERGED['logs-system.auth-default'].close %} +{%- set cur_close_days = CURATORMERGED['logs-system-auth-default'].close %} actions: 1: action: close From bab40de58d7becd7e71059cc01fa5933ac36bf32 Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 13 Mar 2023 17:26:05 -0400 Subject: [PATCH 15/22] Fix system auth default key value --- salt/curator/files/action/logs-system-auth-default-delete.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/curator/files/action/logs-system-auth-default-delete.yaml b/salt/curator/files/action/logs-system-auth-default-delete.yaml index d14d560f3..9a1cc6a9a 100644 --- a/salt/curator/files/action/logs-system-auth-default-delete.yaml +++ b/salt/curator/files/action/logs-system-auth-default-delete.yaml @@ -3,7 +3,7 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -{%- set DELETE_DAYS = CURATORMERGED['logs-system.auth-default'].delete %} +{%- set DELETE_DAYS = CURATORMERGED['logs-system-auth-default'].delete %} actions: 1: action: delete_indices From f4112b30c0402bdca6a5711a48bff4c88f4e1473 Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 13 Mar 2023 17:27:06 -0400 Subject: [PATCH 16/22] Fix index reference for system auth default --- salt/curator/files/action/logs-system-auth-syslog-close.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/curator/files/action/logs-system-auth-syslog-close.yaml b/salt/curator/files/action/logs-system-auth-syslog-close.yaml index 52ddb5eb5..f71ffacb5 100644 --- a/salt/curator/files/action/logs-system-auth-syslog-close.yaml +++ b/salt/curator/files/action/logs-system-auth-syslog-close.yaml @@ -3,7 +3,7 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -{%- set cur_close_days = CURATORMERGED['logs-import-so'].close %} +{%- set cur_close_days = CURATORMERGED['logs-system-auth-default'].close %} actions: 1: action: close @@ -17,7 +17,7 @@ actions: filters: - filtertype: pattern kind: regex - value: '^(.ds-logs-import-so.*)$' + value: '^(.ds-logs-system.auth-default.*)$' - filtertype: age source: name direction: older From 486de12ca5eaee9ecbb9c43dbdab7f73db18a476 Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 13 Mar 2023 17:27:52 -0400 Subject: [PATCH 17/22] Delete logs-system-auth-syslog-close.yaml --- .../action/logs-system-auth-syslog-close.yaml | 27 ------------------- 1 file changed, 27 deletions(-) delete mode 100644 salt/curator/files/action/logs-system-auth-syslog-close.yaml diff --git a/salt/curator/files/action/logs-system-auth-syslog-close.yaml b/salt/curator/files/action/logs-system-auth-syslog-close.yaml deleted file mode 100644 index f71ffacb5..000000000 --- a/salt/curator/files/action/logs-system-auth-syslog-close.yaml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{%- set cur_close_days = CURATORMERGED['logs-system-auth-default'].close %} -actions: - 1: - action: close - description: >- - Close import indices older than {{cur_close_days}} days. - options: - delete_aliases: False - timeout_override: - continue_if_exception: False - disable_action: False - filters: - - filtertype: pattern - kind: regex - value: '^(.ds-logs-system.auth-default.*)$' - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: {{cur_close_days}} - exclude: From 412e5c0402745ee6c287f476c8613ac37c54c64c Mon Sep 17 00:00:00 2001 From: Wes Date: Tue, 14 Mar 2023 13:46:08 +0000 Subject: [PATCH 18/22] Add more Elastic Agent Curator action files --- ...logs-system-application-default-close.yaml | 27 +++++++++++++++++++ ...ogs-system-application-default-delete.yaml | 27 +++++++++++++++++++ .../logs-system-security-default-close.yaml | 27 +++++++++++++++++++ .../logs-system-security-default-delete.yaml | 27 +++++++++++++++++++ .../logs-system-system-default-close.yaml | 27 +++++++++++++++++++ .../logs-system-system-default-delete.yaml | 27 +++++++++++++++++++ ...logs-windows-powershell-default-close.yaml | 27 +++++++++++++++++++ ...ogs-windows-powershell-default-delete.yaml | 27 +++++++++++++++++++ ...dows-sysmon_operational-default-close.yaml | 27 +++++++++++++++++++ ...ows-sysmon_operational-default-delete.yaml | 27 +++++++++++++++++++ 10 files changed, 270 insertions(+) create mode 100644 salt/curator/files/action/logs-system-application-default-close.yaml create mode 100644 salt/curator/files/action/logs-system-application-default-delete.yaml create mode 100644 salt/curator/files/action/logs-system-security-default-close.yaml create mode 100644 salt/curator/files/action/logs-system-security-default-delete.yaml create mode 100644 salt/curator/files/action/logs-system-system-default-close.yaml create mode 100644 salt/curator/files/action/logs-system-system-default-delete.yaml create mode 100644 salt/curator/files/action/logs-windows-powershell-default-close.yaml create mode 100644 salt/curator/files/action/logs-windows-powershell-default-delete.yaml create mode 100644 salt/curator/files/action/logs-windows-sysmon_operational-default-close.yaml create mode 100644 salt/curator/files/action/logs-windows-sysmon_operational-default-delete.yaml diff --git a/salt/curator/files/action/logs-system-application-default-close.yaml b/salt/curator/files/action/logs-system-application-default-close.yaml new file mode 100644 index 000000000..76d01ecb4 --- /dev/null +++ b/salt/curator/files/action/logs-system-application-default-close.yaml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set cur_close_days = CURATORMERGED['logs-system-application-default'].close %} +actions: + 1: + action: close + description: >- + Close Elastic Agent system application indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-system.application-default.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/logs-system-application-default-delete.yaml b/salt/curator/files/action/logs-system-application-default-delete.yaml new file mode 100644 index 000000000..b15c06fcb --- /dev/null +++ b/salt/curator/files/action/logs-system-application-default-delete.yaml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set DELETE_DAYS = CURATORMERGED['logs-system-application-default'].delete %} +actions: + 1: + action: delete_indices + description: >- + Delete Elastic Agent system application indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-system.application-default.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + diff --git a/salt/curator/files/action/logs-system-security-default-close.yaml b/salt/curator/files/action/logs-system-security-default-close.yaml new file mode 100644 index 000000000..9a8cab35c --- /dev/null +++ b/salt/curator/files/action/logs-system-security-default-close.yaml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set cur_close_days = CURATORMERGED['logs-system-security-default'].close %} +actions: + 1: + action: close + description: >- + Close Elastic Agent system security indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-system.security-default.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/logs-system-security-default-delete.yaml b/salt/curator/files/action/logs-system-security-default-delete.yaml new file mode 100644 index 000000000..0bac45aeb --- /dev/null +++ b/salt/curator/files/action/logs-system-security-default-delete.yaml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set DELETE_DAYS = CURATORMERGED['logs-system-security-default'].delete %} +actions: + 1: + action: delete_indices + description: >- + Delete Elastic Agent system security indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-system.security-default.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + diff --git a/salt/curator/files/action/logs-system-system-default-close.yaml b/salt/curator/files/action/logs-system-system-default-close.yaml new file mode 100644 index 000000000..284d6e219 --- /dev/null +++ b/salt/curator/files/action/logs-system-system-default-close.yaml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set cur_close_days = CURATORMERGED['logs-system-system-default'].close %} +actions: + 1: + action: close + description: >- + Close Elastic Agent system system indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-system.system-default.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/logs-system-system-default-delete.yaml b/salt/curator/files/action/logs-system-system-default-delete.yaml new file mode 100644 index 000000000..4701d0492 --- /dev/null +++ b/salt/curator/files/action/logs-system-system-default-delete.yaml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set DELETE_DAYS = CURATORMERGED['logs-system-system-default'].delete %} +actions: + 1: + action: delete_indices + description: >- + Delete Elastic Agent system system indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-system.system-default.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + diff --git a/salt/curator/files/action/logs-windows-powershell-default-close.yaml b/salt/curator/files/action/logs-windows-powershell-default-close.yaml new file mode 100644 index 000000000..7c3cebab3 --- /dev/null +++ b/salt/curator/files/action/logs-windows-powershell-default-close.yaml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set cur_close_days = CURATORMERGED['logs-windows-powershell-default'].close %} +actions: + 1: + action: close + description: >- + Close Elastic Agent Windows Powershell indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-windows.powershell-default.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/logs-windows-powershell-default-delete.yaml b/salt/curator/files/action/logs-windows-powershell-default-delete.yaml new file mode 100644 index 000000000..447f8102b --- /dev/null +++ b/salt/curator/files/action/logs-windows-powershell-default-delete.yaml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set DELETE_DAYS = CURATORMERGED['logs-windows-powershell-default'].delete %} +actions: + 1: + action: delete_indices + description: >- + Delete Elastic Agent Windows Powershell indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-windows.powershell-default.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + diff --git a/salt/curator/files/action/logs-windows-sysmon_operational-default-close.yaml b/salt/curator/files/action/logs-windows-sysmon_operational-default-close.yaml new file mode 100644 index 000000000..ae98b8939 --- /dev/null +++ b/salt/curator/files/action/logs-windows-sysmon_operational-default-close.yaml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set cur_close_days = CURATORMERGED['logs-windows-sysmon_operational-default'].close %} +actions: + 1: + action: close + description: >- + Close Elastic Agent Windows Sysmon operational indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-windows.sysmon_operational-default.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/logs-windows-sysmon_operational-default-delete.yaml b/salt/curator/files/action/logs-windows-sysmon_operational-default-delete.yaml new file mode 100644 index 000000000..9a1cc6a9a --- /dev/null +++ b/salt/curator/files/action/logs-windows-sysmon_operational-default-delete.yaml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set DELETE_DAYS = CURATORMERGED['logs-system-auth-default'].delete %} +actions: + 1: + action: delete_indices + description: >- + Delete Elastic Agent system auth indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-system.auth-default.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + From f0d4c16b2ba35a5a7f990b4f4d2e51fdc091ebbd Mon Sep 17 00:00:00 2001 From: Wes Date: Tue, 14 Mar 2023 13:49:13 +0000 Subject: [PATCH 19/22] Add more Elastic Agent index keys for Curator --- salt/curator/defaults.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/salt/curator/defaults.yaml b/salt/curator/defaults.yaml index 958dd99ef..e1333c3a6 100644 --- a/salt/curator/defaults.yaml +++ b/salt/curator/defaults.yaml @@ -33,9 +33,24 @@ elasticsearch: logs-system-auth-default: close: 30 delete: 365 + logs-system-application-default: + close: 30 + delete: 365 + logs-system-security-default: + close: 30 + delete: 365 + logs-system-system-default: + close: 30 + delete: 365 logs-system-syslog-default: close: 30 delete: 365 + logs-windows-powershell-default: + close: 30 + delete: 365 + logs-windows-sysmon_operational-default: + close: 30 + delete: 365 so-beats: close: 30 delete: 365 From 766e6a79745671dc0cffad8d7c7f92d3071326fc Mon Sep 17 00:00:00 2001 From: Wes Date: Tue, 14 Mar 2023 13:51:49 +0000 Subject: [PATCH 20/22] Add 'logs-windows-sysmon_operational-delete' for Windows Sysmon operational indices --- .../logs-windows-sysmon_operational-default-delete.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/curator/files/action/logs-windows-sysmon_operational-default-delete.yaml b/salt/curator/files/action/logs-windows-sysmon_operational-default-delete.yaml index 9a1cc6a9a..a1413bc1c 100644 --- a/salt/curator/files/action/logs-windows-sysmon_operational-default-delete.yaml +++ b/salt/curator/files/action/logs-windows-sysmon_operational-default-delete.yaml @@ -3,19 +3,19 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -{%- set DELETE_DAYS = CURATORMERGED['logs-system-auth-default'].delete %} +{%- set DELETE_DAYS = CURATORMERGED['logs-windows-sysmon_operational-default'].delete %} actions: 1: action: delete_indices description: >- - Delete Elastic Agent system auth indices when older than {{ DELETE_DAYS }} days. + Delete Elastic Agent Windows Sysmon operational indices when older than {{ DELETE_DAYS }} days. options: ignore_empty_list: True disable_action: False filters: - filtertype: pattern kind: regex - value: '^(.ds-logs-system.auth-default.*)$' + value: '^(.ds-logs-windows.sysmon_operational-default.*)$' - filtertype: age source: name direction: older From 7c39938e14b8d8c87484d46ad5890fbf5ffff2b8 Mon Sep 17 00:00:00 2001 From: weslambert Date: Tue, 14 Mar 2023 10:48:50 -0400 Subject: [PATCH 21/22] Change 'elastic_agent.filebeat' to 'elastic_agent-filebeat' --- .../files/action/logs-elastic_agent-filebeat-default-close.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/curator/files/action/logs-elastic_agent-filebeat-default-close.yaml b/salt/curator/files/action/logs-elastic_agent-filebeat-default-close.yaml index 9277b25fd..1157f94b2 100644 --- a/salt/curator/files/action/logs-elastic_agent-filebeat-default-close.yaml +++ b/salt/curator/files/action/logs-elastic_agent-filebeat-default-close.yaml @@ -3,7 +3,7 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -{%- set cur_close_days = CURATORMERGED['logs-elastic_agent.filebeat-default'].close %} +{%- set cur_close_days = CURATORMERGED['logs-elastic_agent-filebeat-default'].close %} actions: 1: action: close From 8eba3426be104d34dc73247aa2e0ede293cda78e Mon Sep 17 00:00:00 2001 From: weslambert Date: Tue, 14 Mar 2023 10:51:50 -0400 Subject: [PATCH 22/22] Remove extra dash for 'logs-elastic_agent-metricbeat-default' key --- .../action/logs-elastic_agent-metricbeat-default-delete.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/curator/files/action/logs-elastic_agent-metricbeat-default-delete.yaml b/salt/curator/files/action/logs-elastic_agent-metricbeat-default-delete.yaml index b42e42c83..c69e1130a 100644 --- a/salt/curator/files/action/logs-elastic_agent-metricbeat-default-delete.yaml +++ b/salt/curator/files/action/logs-elastic_agent-metricbeat-default-delete.yaml @@ -3,7 +3,7 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -{%- set DELETE_DAYS = CURATORMERGED['logs--elastic_agent-metricbeat-default'].delete %} +{%- set DELETE_DAYS = CURATORMERGED['logs-elastic_agent-metricbeat-default'].delete %} actions: 1: action: delete_indices