From d6365468716ba93a49939a73b152fcb2a35301c8 Mon Sep 17 00:00:00 2001 From: Wes Date: Tue, 7 Mar 2023 17:15:25 +0000 Subject: [PATCH 1/4] Add new Curator action files --- .../files/action/logs-import-so-close.yml | 27 +++++++++++++++++++ .../files/action/logs-import-so-delete.yml | 27 +++++++++++++++++++ .../files/action/logs-strelka-so-close.yml | 27 +++++++++++++++++++ .../files/action/logs-strelka-so-delete.yml | 27 +++++++++++++++++++ .../files/action/logs-suricata-so-close.yml | 27 +++++++++++++++++++ .../files/action/logs-suricata-so-delete.yml | 27 +++++++++++++++++++ .../files/action/logs-syslog-so-close.yml | 27 +++++++++++++++++++ .../files/action/logs-syslog-so-delete.yml | 27 +++++++++++++++++++ .../files/action/logs-zeek-so-close.yml | 27 +++++++++++++++++++ .../files/action/logs-zeek-so-delete.yml | 27 +++++++++++++++++++ 10 files changed, 270 insertions(+) create mode 100644 salt/curator/files/action/logs-import-so-close.yml create mode 100644 salt/curator/files/action/logs-import-so-delete.yml create mode 100644 salt/curator/files/action/logs-strelka-so-close.yml create mode 100644 salt/curator/files/action/logs-strelka-so-delete.yml create mode 100644 salt/curator/files/action/logs-suricata-so-close.yml create mode 100644 salt/curator/files/action/logs-suricata-so-delete.yml create mode 100644 salt/curator/files/action/logs-syslog-so-close.yml create mode 100644 salt/curator/files/action/logs-syslog-so-delete.yml create mode 100644 salt/curator/files/action/logs-zeek-so-close.yml create mode 100644 salt/curator/files/action/logs-zeek-so-delete.yml diff --git a/salt/curator/files/action/logs-import-so-close.yml b/salt/curator/files/action/logs-import-so-close.yml new file mode 100644 index 000000000..52ddb5eb5 --- /dev/null +++ b/salt/curator/files/action/logs-import-so-close.yml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set cur_close_days = CURATORMERGED['logs-import-so'].close %} +actions: + 1: + action: close + description: >- + Close import indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-import-so.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/logs-import-so-delete.yml b/salt/curator/files/action/logs-import-so-delete.yml new file mode 100644 index 000000000..274d06711 --- /dev/null +++ b/salt/curator/files/action/logs-import-so-delete.yml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set DELETE_DAYS = CURATORMERGED['logs-syslog-so'].delete %} +actions: + 1: + action: delete_indices + description: >- + Delete syslog indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-syslog-so.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + diff --git a/salt/curator/files/action/logs-strelka-so-close.yml b/salt/curator/files/action/logs-strelka-so-close.yml new file mode 100644 index 000000000..a5b31785f --- /dev/null +++ b/salt/curator/files/action/logs-strelka-so-close.yml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set cur_close_days = CURATORMERGED['logs-strelka-so'].close %} +actions: + 1: + action: close + description: >- + Close Strelka indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-strelka-so.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/logs-strelka-so-delete.yml b/salt/curator/files/action/logs-strelka-so-delete.yml new file mode 100644 index 000000000..d01bdcc83 --- /dev/null +++ b/salt/curator/files/action/logs-strelka-so-delete.yml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set DELETE_DAYS = CURATORMERGED['logs-strelka-so'].delete %} +actions: + 1: + action: delete_indices + description: >- + Delete Strelka indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-strelka-so.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + diff --git a/salt/curator/files/action/logs-suricata-so-close.yml b/salt/curator/files/action/logs-suricata-so-close.yml new file mode 100644 index 000000000..a25be9f3d --- /dev/null +++ b/salt/curator/files/action/logs-suricata-so-close.yml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set cur_close_days = CURATORMERGED['logs-suricata-so'].close %} +actions: + 1: + action: close + description: >- + Close Suricata indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-suricata-so.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/logs-suricata-so-delete.yml b/salt/curator/files/action/logs-suricata-so-delete.yml new file mode 100644 index 000000000..765ba1293 --- /dev/null +++ b/salt/curator/files/action/logs-suricata-so-delete.yml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set DELETE_DAYS = CURATORMERGED['logs-suricata-so'].delete %} +actions: + 1: + action: delete_indices + description: >- + Delete Suricata indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-suricata-so.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + diff --git a/salt/curator/files/action/logs-syslog-so-close.yml b/salt/curator/files/action/logs-syslog-so-close.yml new file mode 100644 index 000000000..b9baf3c1a --- /dev/null +++ b/salt/curator/files/action/logs-syslog-so-close.yml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set cur_close_days = CURATORMERGED['logs-syslog-so'].close %} +actions: + 1: + action: close + description: >- + Close syslog indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-syslog-so.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/logs-syslog-so-delete.yml b/salt/curator/files/action/logs-syslog-so-delete.yml new file mode 100644 index 000000000..b46a5fc73 --- /dev/null +++ b/salt/curator/files/action/logs-syslog-so-delete.yml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set DELETE_DAYS = CURATORMERGED['logs-import-so'].delete %} +actions: + 1: + action: delete_indices + description: >- + Delete import indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-import-so.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + diff --git a/salt/curator/files/action/logs-zeek-so-close.yml b/salt/curator/files/action/logs-zeek-so-close.yml new file mode 100644 index 000000000..f8ad13ca0 --- /dev/null +++ b/salt/curator/files/action/logs-zeek-so-close.yml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set cur_close_days = CURATORMERGED['logs-zeek-so'].close %} +actions: + 1: + action: close + description: >- + Close Zeek indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-zeek-so.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/logs-zeek-so-delete.yml b/salt/curator/files/action/logs-zeek-so-delete.yml new file mode 100644 index 000000000..5acfc50a7 --- /dev/null +++ b/salt/curator/files/action/logs-zeek-so-delete.yml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set DELETE_DAYS = CURATORMERGED['logs-zeek-so'].delete %} +actions: + 1: + action: delete_indices + description: >- + Delete Zeek indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-zeek-so.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + From 88d98af243ef5bf19024e2c3b7abc02b15d9aa2a Mon Sep 17 00:00:00 2001 From: Wes Date: Tue, 7 Mar 2023 17:21:03 +0000 Subject: [PATCH 2/4] Add new Curator action files to Curator close and delete scripts --- salt/curator/files/bin/so-curator-close | 5 +++++ salt/curator/files/bin/so-curator-cluster-close | 5 +++++ salt/curator/files/bin/so-curator-cluster-delete | 5 +++++ 3 files changed, 15 insertions(+) diff --git a/salt/curator/files/bin/so-curator-close b/salt/curator/files/bin/so-curator-close index 885cb4502..af66a03df 100644 --- a/salt/curator/files/bin/so-curator-close +++ b/salt/curator/files/bin/so-curator-close @@ -25,3 +25,8 @@ docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/cur docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-ossec-close.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-strelka-close.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-syslog-close.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/logs-import-so-close.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/logs-strelka-close.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/logs-suricata-close.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/logs-syslog-close.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/logs-zeek-close.yml > /dev/null 2>&1; diff --git a/salt/curator/files/bin/so-curator-cluster-close b/salt/curator/files/bin/so-curator-cluster-close index 0929149ed..4359dcfc1 100644 --- a/salt/curator/files/bin/so-curator-cluster-close +++ b/salt/curator/files/bin/so-curator-cluster-close @@ -23,3 +23,8 @@ docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/cur docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-ossec-close.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-strelka-close.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-syslog-close.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/logs-import-so-close.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/logs-strelka-close.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/logs-suricata-close.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/logs-syslog-close.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/logs-zeek-close.yml > /dev/null 2>&1; diff --git a/salt/curator/files/bin/so-curator-cluster-delete b/salt/curator/files/bin/so-curator-cluster-delete index 2d71f725d..34c3c10cf 100644 --- a/salt/curator/files/bin/so-curator-cluster-delete +++ b/salt/curator/files/bin/so-curator-cluster-delete @@ -23,3 +23,8 @@ docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/cur docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-ossec-delete.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-strelka-delete.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-syslog-delete.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/logs-import-so-delete.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/logs-strelka-delete.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/logs-suricata-delete.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/logs-syslog-delete.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/logs-zeek-delete.yml > /dev/null 2>&1; From 26c9813276675f9646fbea13753e6e8c3379fca5 Mon Sep 17 00:00:00 2001 From: Wes Date: Tue, 7 Mar 2023 17:29:07 +0000 Subject: [PATCH 3/4] Add keys for new Curator actions to defaults.yaml --- salt/curator/defaults.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/salt/curator/defaults.yaml b/salt/curator/defaults.yaml index 8e791b0d5..237a50c81 100644 --- a/salt/curator/defaults.yaml +++ b/salt/curator/defaults.yaml @@ -1,5 +1,20 @@ elasticsearch: index_settings: + logs-import-so: + close: 73000 + delete: 73001 + logs-strelka-so: + close: 30 + delete: 365 + logs-suricata-so: + close: 30 + delete: 365 + logs-syslog-so: + close: 30 + delete: 365 + logs-zeek-so: + close: 30 + delete: 365 so-beats: close: 30 delete: 365 From f50639d2d2c6950492439ce110e78a160ee8af66 Mon Sep 17 00:00:00 2001 From: Wes Date: Tue, 7 Mar 2023 17:41:48 +0000 Subject: [PATCH 4/4] Fix import and syslog actions --- salt/curator/files/action/logs-import-so-delete.yml | 6 +++--- salt/curator/files/action/logs-syslog-so-delete.yml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/salt/curator/files/action/logs-import-so-delete.yml b/salt/curator/files/action/logs-import-so-delete.yml index 274d06711..b46a5fc73 100644 --- a/salt/curator/files/action/logs-import-so-delete.yml +++ b/salt/curator/files/action/logs-import-so-delete.yml @@ -3,19 +3,19 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -{%- set DELETE_DAYS = CURATORMERGED['logs-syslog-so'].delete %} +{%- set DELETE_DAYS = CURATORMERGED['logs-import-so'].delete %} actions: 1: action: delete_indices description: >- - Delete syslog indices when older than {{ DELETE_DAYS }} days. + Delete import indices when older than {{ DELETE_DAYS }} days. options: ignore_empty_list: True disable_action: False filters: - filtertype: pattern kind: regex - value: '^(.ds-logs-syslog-so.*)$' + value: '^(.ds-logs-import-so.*)$' - filtertype: age source: name direction: older diff --git a/salt/curator/files/action/logs-syslog-so-delete.yml b/salt/curator/files/action/logs-syslog-so-delete.yml index b46a5fc73..274d06711 100644 --- a/salt/curator/files/action/logs-syslog-so-delete.yml +++ b/salt/curator/files/action/logs-syslog-so-delete.yml @@ -3,19 +3,19 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -{%- set DELETE_DAYS = CURATORMERGED['logs-import-so'].delete %} +{%- set DELETE_DAYS = CURATORMERGED['logs-syslog-so'].delete %} actions: 1: action: delete_indices description: >- - Delete import indices when older than {{ DELETE_DAYS }} days. + Delete syslog indices when older than {{ DELETE_DAYS }} days. options: ignore_empty_list: True disable_action: False filters: - filtertype: pattern kind: regex - value: '^(.ds-logs-import-so.*)$' + value: '^(.ds-logs-syslog-so.*)$' - filtertype: age source: name direction: older