From 8d395dc465911918c3a2633bffb58199b524e7fa Mon Sep 17 00:00:00 2001 From: Wes Date: Mon, 13 Mar 2023 20:54:13 +0000 Subject: [PATCH 1/2] Add Elastic Agent default data stream backing indices for management by Curator --- salt/curator/defaults.yaml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/salt/curator/defaults.yaml b/salt/curator/defaults.yaml index 237a50c81..958dd99ef 100644 --- a/salt/curator/defaults.yaml +++ b/salt/curator/defaults.yaml @@ -15,6 +15,27 @@ elasticsearch: logs-zeek-so: close: 30 delete: 365 + logs-elastic_agent-metricbeat-default: + close: 30 + delete: 365 + logs-elastic_agent-osquerybeat-default: + close: 30 + delete: 365 + logs-elastic_agent-fleet_server-default: + close: 30 + delete: 365 + logs-elastic_agent-filebeat-default: + close: 30 + delete: 365 + logs-elastic_agent-default: + close: 30 + delete: 365 + logs-system-auth-default: + close: 30 + delete: 365 + logs-system-syslog-default: + close: 30 + delete: 365 so-beats: close: 30 delete: 365 From efc58324999253c158915e710154206ffc671988 Mon Sep 17 00:00:00 2001 From: Wes Date: Mon, 13 Mar 2023 20:54:38 +0000 Subject: [PATCH 2/2] Add Elastic Agent default log action files --- .../logs-elastic_agent-default-close.yaml | 27 +++++++++++++++++++ .../logs-elastic_agent-default-delete.yaml | 27 +++++++++++++++++++ ...-elastic_agent-filebeat-default-close.yaml | 27 +++++++++++++++++++ ...elastic_agent-filebeat-default-delete.yaml | 27 +++++++++++++++++++ ...stic_agent-fleet_server-default-close.yaml | 27 +++++++++++++++++++ ...tic_agent-fleet_server-default-delete.yaml | 27 +++++++++++++++++++ ...lastic_agent-metricbeat-default-close.yaml | 27 +++++++++++++++++++ ...astic_agent-metricbeat-default-delete.yaml | 27 +++++++++++++++++++ ...astic_agent-osquerybeat-default-close.yaml | 27 +++++++++++++++++++ ...stic_agent-osquerybeat-default-delete.yaml | 27 +++++++++++++++++++ ...logs-elastic_agent-osquerybeat-delete.yaml | 27 +++++++++++++++++++ .../logs-system-auth-default-close.yaml | 27 +++++++++++++++++++ .../logs-system-auth-default-delete.yaml | 27 +++++++++++++++++++ .../action/logs-system-auth-syslog-close.yaml | 27 +++++++++++++++++++ .../logs-system-syslog-default-close.yaml | 27 +++++++++++++++++++ .../logs-system-syslog-default-delete.yaml | 27 +++++++++++++++++++ 16 files changed, 432 insertions(+) create mode 100644 salt/curator/files/action/logs-elastic_agent-default-close.yaml create mode 100644 salt/curator/files/action/logs-elastic_agent-default-delete.yaml create mode 100644 salt/curator/files/action/logs-elastic_agent-filebeat-default-close.yaml create mode 100644 salt/curator/files/action/logs-elastic_agent-filebeat-default-delete.yaml create mode 100644 salt/curator/files/action/logs-elastic_agent-fleet_server-default-close.yaml create mode 100644 salt/curator/files/action/logs-elastic_agent-fleet_server-default-delete.yaml create mode 100644 salt/curator/files/action/logs-elastic_agent-metricbeat-default-close.yaml create mode 100644 salt/curator/files/action/logs-elastic_agent-metricbeat-default-delete.yaml create mode 100644 salt/curator/files/action/logs-elastic_agent-osquerybeat-default-close.yaml create mode 100644 salt/curator/files/action/logs-elastic_agent-osquerybeat-default-delete.yaml create mode 100644 salt/curator/files/action/logs-elastic_agent-osquerybeat-delete.yaml create mode 100644 salt/curator/files/action/logs-system-auth-default-close.yaml create mode 100644 salt/curator/files/action/logs-system-auth-default-delete.yaml create mode 100644 salt/curator/files/action/logs-system-auth-syslog-close.yaml create mode 100644 salt/curator/files/action/logs-system-syslog-default-close.yaml create mode 100644 salt/curator/files/action/logs-system-syslog-default-delete.yaml diff --git a/salt/curator/files/action/logs-elastic_agent-default-close.yaml b/salt/curator/files/action/logs-elastic_agent-default-close.yaml new file mode 100644 index 000000000..ef03e4ba2 --- /dev/null +++ b/salt/curator/files/action/logs-elastic_agent-default-close.yaml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set cur_close_days = CURATORMERGED['logs-elastic_agent-default'].close %} +actions: + 1: + action: close + description: >- + Close Elastic Agent default indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-elastic_agent-default.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/logs-elastic_agent-default-delete.yaml b/salt/curator/files/action/logs-elastic_agent-default-delete.yaml new file mode 100644 index 000000000..dee51c758 --- /dev/null +++ b/salt/curator/files/action/logs-elastic_agent-default-delete.yaml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set DELETE_DAYS = CURATORMERGED['logs-elastic_agent-default'].delete %} +actions: + 1: + action: delete_indices + description: >- + Delete Elastic Agent default indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-elastic_agent-default.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + diff --git a/salt/curator/files/action/logs-elastic_agent-filebeat-default-close.yaml b/salt/curator/files/action/logs-elastic_agent-filebeat-default-close.yaml new file mode 100644 index 000000000..9277b25fd --- /dev/null +++ b/salt/curator/files/action/logs-elastic_agent-filebeat-default-close.yaml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set cur_close_days = CURATORMERGED['logs-elastic_agent.filebeat-default'].close %} +actions: + 1: + action: close + description: >- + Close Elastic Agent Filebeat indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-elastic_agent.filebeat-default.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/logs-elastic_agent-filebeat-default-delete.yaml b/salt/curator/files/action/logs-elastic_agent-filebeat-default-delete.yaml new file mode 100644 index 000000000..dfa51f260 --- /dev/null +++ b/salt/curator/files/action/logs-elastic_agent-filebeat-default-delete.yaml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set DELETE_DAYS = CURATORMERGED['logs-elastic_agent-filebeat-default'].delete %} +actions: + 1: + action: delete_indices + description: >- + Delete Elastic Agent Filebeat indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-elastic_agent.filebeat-default.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + diff --git a/salt/curator/files/action/logs-elastic_agent-fleet_server-default-close.yaml b/salt/curator/files/action/logs-elastic_agent-fleet_server-default-close.yaml new file mode 100644 index 000000000..6bc2026b9 --- /dev/null +++ b/salt/curator/files/action/logs-elastic_agent-fleet_server-default-close.yaml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set cur_close_days = CURATORMERGED['logs-elastic_agent-fleet_server-default'].close %} +actions: + 1: + action: close + description: >- + Close Elastic Agent Fleet Server indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-elastic_agent.fleet_server-default.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/logs-elastic_agent-fleet_server-default-delete.yaml b/salt/curator/files/action/logs-elastic_agent-fleet_server-default-delete.yaml new file mode 100644 index 000000000..6fa775ba8 --- /dev/null +++ b/salt/curator/files/action/logs-elastic_agent-fleet_server-default-delete.yaml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set DELETE_DAYS = CURATORMERGED['logs-elastic_agent-fleet_server-default'].delete %} +actions: + 1: + action: delete_indices + description: >- + Delete import indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-elastic_agent.fleet_server-default.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + diff --git a/salt/curator/files/action/logs-elastic_agent-metricbeat-default-close.yaml b/salt/curator/files/action/logs-elastic_agent-metricbeat-default-close.yaml new file mode 100644 index 000000000..a4e38cd8e --- /dev/null +++ b/salt/curator/files/action/logs-elastic_agent-metricbeat-default-close.yaml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set cur_close_days = CURATORMERGED['logs-elastic_agent-metricbeat-default'].close %} +actions: + 1: + action: close + description: >- + Close Elastic Agent Metricbeat indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-elastic_agent.metricbeat-default-.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/logs-elastic_agent-metricbeat-default-delete.yaml b/salt/curator/files/action/logs-elastic_agent-metricbeat-default-delete.yaml new file mode 100644 index 000000000..b42e42c83 --- /dev/null +++ b/salt/curator/files/action/logs-elastic_agent-metricbeat-default-delete.yaml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set DELETE_DAYS = CURATORMERGED['logs--elastic_agent-metricbeat-default'].delete %} +actions: + 1: + action: delete_indices + description: >- + Delete Elastic Agent Metricbeat indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-elastic_agent.metricbeat-default.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + diff --git a/salt/curator/files/action/logs-elastic_agent-osquerybeat-default-close.yaml b/salt/curator/files/action/logs-elastic_agent-osquerybeat-default-close.yaml new file mode 100644 index 000000000..9243d8cfb --- /dev/null +++ b/salt/curator/files/action/logs-elastic_agent-osquerybeat-default-close.yaml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set cur_close_days = CURATORMERGED['logs-elastic_agent-osquerybeat-default'].close %} +actions: + 1: + action: close + description: >- + Close Elastic Agent Osquerybeat indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-elastic_agent.osquerybeat-default.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/logs-elastic_agent-osquerybeat-default-delete.yaml b/salt/curator/files/action/logs-elastic_agent-osquerybeat-default-delete.yaml new file mode 100644 index 000000000..bce3b7e63 --- /dev/null +++ b/salt/curator/files/action/logs-elastic_agent-osquerybeat-default-delete.yaml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set DELETE_DAYS = CURATORMERGED['logs-elastic_agent-osquerybeat-default'].delete %} +actions: + 1: + action: delete_indices + description: >- + Delete Elastic Agent Osquerybeat indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-elastic_agent.osquerybeat-default.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + diff --git a/salt/curator/files/action/logs-elastic_agent-osquerybeat-delete.yaml b/salt/curator/files/action/logs-elastic_agent-osquerybeat-delete.yaml new file mode 100644 index 000000000..b46a5fc73 --- /dev/null +++ b/salt/curator/files/action/logs-elastic_agent-osquerybeat-delete.yaml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set DELETE_DAYS = CURATORMERGED['logs-import-so'].delete %} +actions: + 1: + action: delete_indices + description: >- + Delete import indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-import-so.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + diff --git a/salt/curator/files/action/logs-system-auth-default-close.yaml b/salt/curator/files/action/logs-system-auth-default-close.yaml new file mode 100644 index 000000000..7c04a0ca9 --- /dev/null +++ b/salt/curator/files/action/logs-system-auth-default-close.yaml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set cur_close_days = CURATORMERGED['logs-system.auth-default'].close %} +actions: + 1: + action: close + description: >- + Close Elastic Agent system auth indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-system.auth-default.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/logs-system-auth-default-delete.yaml b/salt/curator/files/action/logs-system-auth-default-delete.yaml new file mode 100644 index 000000000..d14d560f3 --- /dev/null +++ b/salt/curator/files/action/logs-system-auth-default-delete.yaml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set DELETE_DAYS = CURATORMERGED['logs-system.auth-default'].delete %} +actions: + 1: + action: delete_indices + description: >- + Delete Elastic Agent system auth indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-system.auth-default.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + diff --git a/salt/curator/files/action/logs-system-auth-syslog-close.yaml b/salt/curator/files/action/logs-system-auth-syslog-close.yaml new file mode 100644 index 000000000..52ddb5eb5 --- /dev/null +++ b/salt/curator/files/action/logs-system-auth-syslog-close.yaml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set cur_close_days = CURATORMERGED['logs-import-so'].close %} +actions: + 1: + action: close + description: >- + Close import indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-import-so.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/logs-system-syslog-default-close.yaml b/salt/curator/files/action/logs-system-syslog-default-close.yaml new file mode 100644 index 000000000..a9a697a66 --- /dev/null +++ b/salt/curator/files/action/logs-system-syslog-default-close.yaml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set cur_close_days = CURATORMERGED['logs-system.syslog-default'].close %} +actions: + 1: + action: close + description: >- + Close Elastic Agent system syslog indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-system.syslog-default.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/logs-system-syslog-default-delete.yaml b/salt/curator/files/action/logs-system-syslog-default-delete.yaml new file mode 100644 index 000000000..b46a5fc73 --- /dev/null +++ b/salt/curator/files/action/logs-system-syslog-default-delete.yaml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set DELETE_DAYS = CURATORMERGED['logs-import-so'].delete %} +actions: + 1: + action: delete_indices + description: >- + Delete import indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-import-so.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + +