mirror of
https://github.com/Security-Onion-Solutions/securityonion.git
synced 2026-05-01 00:47:54 +02:00
Remove Curator
This commit is contained in:
Wes
@@ -1,81 +0,0 @@
|
||||
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
|
||||
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
|
||||
# https://securityonion.net/license; you may not use this file except in compliance with the
|
||||
# Elastic License 2.0.
|
||||
|
||||
{% from 'allowed_states.map.jinja' import allowed_states %}
|
||||
{% if sls.split('.')[0] in allowed_states %}
|
||||
{% from "curator/map.jinja" import CURATORMERGED %}
|
||||
|
||||
# Create the group
|
||||
curatorgroup:
|
||||
group.present:
|
||||
- name: curator
|
||||
- gid: 934
|
||||
|
||||
# Add user
|
||||
curator:
|
||||
user.present:
|
||||
- uid: 934
|
||||
- gid: 934
|
||||
- home: /opt/so/conf/curator
|
||||
- createhome: False
|
||||
|
||||
# Create the log directory
|
||||
curlogdir:
|
||||
file.directory:
|
||||
- name: /opt/so/log/curator
|
||||
- user: 934
|
||||
- group: 939
|
||||
|
||||
curactiondir:
|
||||
file.directory:
|
||||
- name: /opt/so/conf/curator/action
|
||||
- user: 934
|
||||
- group: 939
|
||||
- makedirs: True
|
||||
|
||||
actionconfs:
|
||||
file.recurse:
|
||||
- name: /opt/so/conf/curator/action
|
||||
- source: salt://curator/files/action
|
||||
- user: 934
|
||||
- group: 939
|
||||
- template: jinja
|
||||
- defaults:
|
||||
CURATORMERGED: {{ CURATORMERGED.elasticsearch.index_settings }}
|
||||
|
||||
curconf:
|
||||
file.managed:
|
||||
- name: /opt/so/conf/curator/curator.yml
|
||||
- source: salt://curator/files/curator.yml
|
||||
- user: 934
|
||||
- group: 939
|
||||
- mode: 660
|
||||
- template: jinja
|
||||
- show_changes: False
|
||||
|
||||
curator_sbin:
|
||||
file.recurse:
|
||||
- name: /usr/sbin
|
||||
- source: salt://curator/tools/sbin
|
||||
- user: 934
|
||||
- group: 939
|
||||
- file_mode: 755
|
||||
|
||||
curator_sbin_jinja:
|
||||
file.recurse:
|
||||
- name: /usr/sbin
|
||||
- source: salt://curator/tools/sbin_jinja
|
||||
- user: 934
|
||||
- group: 939
|
||||
- file_mode: 755
|
||||
- template: jinja
|
||||
|
||||
{% else %}
|
||||
|
||||
{{sls}}_state_not_allowed:
|
||||
test.fail_without_changes:
|
||||
- name: {{sls}}_state_not_allowed
|
||||
|
||||
{% endif %}
|
||||
@@ -1,34 +0,0 @@
|
||||
curator:
|
||||
enabled: False
|
||||
elasticsearch:
|
||||
index_settings:
|
||||
so-beats:
|
||||
delete: 365
|
||||
so-elasticsearch:
|
||||
delete: 365
|
||||
so-firewall:
|
||||
delete: 365
|
||||
so-ids:
|
||||
delete: 365
|
||||
so-import:
|
||||
delete: 73001
|
||||
so-kratos:
|
||||
delete: 365
|
||||
so-kibana:
|
||||
delete: 365
|
||||
so-logstash:
|
||||
delete: 365
|
||||
so-netflow:
|
||||
delete: 365
|
||||
so-osquery:
|
||||
delete: 365
|
||||
so-ossec:
|
||||
delete: 365
|
||||
so-redis:
|
||||
delete: 365
|
||||
so-strelka:
|
||||
delete: 365
|
||||
so-syslog:
|
||||
delete: 365
|
||||
so-zeek:
|
||||
delete: 365
|
||||
@@ -1,35 +0,0 @@
|
||||
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
|
||||
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
|
||||
# https://securityonion.net/license; you may not use this file except in compliance with the
|
||||
# Elastic License 2.0.
|
||||
|
||||
{% from 'allowed_states.map.jinja' import allowed_states %}
|
||||
{% if sls.split('.')[0] in allowed_states %}
|
||||
|
||||
include:
|
||||
- curator.sostatus
|
||||
|
||||
so-curator:
|
||||
docker_container.absent:
|
||||
- force: True
|
||||
|
||||
so-curator_so-status.disabled:
|
||||
file.comment:
|
||||
- name: /opt/so/conf/so-status/so-status.conf
|
||||
- regex: ^so-curator$
|
||||
|
||||
so-curator-cluster-close:
|
||||
cron.absent:
|
||||
- identifier: so-curator-cluster-close
|
||||
|
||||
so-curator-cluster-delete:
|
||||
cron.absent:
|
||||
- identifier: so-curator-cluster-delete
|
||||
|
||||
{% else %}
|
||||
|
||||
{{sls}}_state_not_allowed:
|
||||
test.fail_without_changes:
|
||||
- name: {{sls}}_state_not_allowed
|
||||
|
||||
{% endif %}
|
||||
@@ -1,81 +0,0 @@
|
||||
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
|
||||
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
|
||||
# https://securityonion.net/license; you may not use this file except in compliance with the
|
||||
# Elastic License 2.0.
|
||||
|
||||
{% from 'allowed_states.map.jinja' import allowed_states %}
|
||||
{% if sls.split('.')[0] in allowed_states %}
|
||||
{% from 'vars/globals.map.jinja' import GLOBALS %}
|
||||
{% from 'docker/docker.map.jinja' import DOCKER %}
|
||||
|
||||
include:
|
||||
- curator.config
|
||||
- curator.sostatus
|
||||
|
||||
so-curator:
|
||||
docker_container.running:
|
||||
- image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-curator:{{ GLOBALS.so_version }}
|
||||
- start: True
|
||||
- hostname: curator
|
||||
- name: so-curator
|
||||
- user: curator
|
||||
- networks:
|
||||
- sobridge:
|
||||
- ipv4_address: {{ DOCKER.containers['so-curator'].ip }}
|
||||
- interactive: True
|
||||
- tty: True
|
||||
- binds:
|
||||
- /opt/so/conf/curator/curator.yml:/etc/curator/config/curator.yml:ro
|
||||
- /opt/so/conf/curator/action/:/etc/curator/action:ro
|
||||
- /opt/so/log/curator:/var/log/curator:rw
|
||||
{% if DOCKER.containers['so-curator'].custom_bind_mounts %}
|
||||
{% for BIND in DOCKER.containers['so-curator'].custom_bind_mounts %}
|
||||
- {{ BIND }}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
{% if DOCKER.containers['so-curator'].extra_hosts %}
|
||||
- extra_hosts:
|
||||
{% for XTRAHOST in DOCKER.containers['so-curator'].extra_hosts %}
|
||||
- {{ XTRAHOST }}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
{% if DOCKER.containers['so-curator'].extra_env %}
|
||||
- environment:
|
||||
{% for XTRAENV in DOCKER.containers['so-curator'].extra_env %}
|
||||
- {{ XTRAENV }}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
- require:
|
||||
- file: actionconfs
|
||||
- file: curconf
|
||||
- file: curlogdir
|
||||
- watch:
|
||||
- file: curconf
|
||||
|
||||
delete_so-curator_so-status.disabled:
|
||||
file.uncomment:
|
||||
- name: /opt/so/conf/so-status/so-status.conf
|
||||
- regex: ^so-curator$
|
||||
|
||||
so-curator-cluster-close:
|
||||
cron.absent:
|
||||
- identifier: so-curator-cluster-close
|
||||
|
||||
so-curator-cluster-delete:
|
||||
cron.present:
|
||||
- name: /usr/sbin/so-curator-cluster-delete > /opt/so/log/curator/cron-cluster-delete.log 2>&1
|
||||
- identifier: so-curator-cluster-delete
|
||||
- user: root
|
||||
- minute: '*/5'
|
||||
- hour: '*'
|
||||
- daymonth: '*'
|
||||
- month: '*'
|
||||
- dayweek: '*'
|
||||
|
||||
{% else %}
|
||||
|
||||
{{sls}}_state_not_allowed:
|
||||
test.fail_without_changes:
|
||||
- name: {{sls}}_state_not_allowed
|
||||
|
||||
{% endif %}
|
||||
@@ -1,32 +0,0 @@
|
||||
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
|
||||
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
|
||||
# https://securityonion.net/license; you may not use this file except in compliance with the
|
||||
# Elastic License 2.0.
|
||||
|
||||
{% import_yaml 'elasticsearch/defaults.yaml' as ELASTICDEFAULTS %}
|
||||
{% set ELASTICMERGED = salt['pillar.get']('elasticsearch:retention', ELASTICDEFAULTS.elasticsearch.retention, merge=true) %}
|
||||
|
||||
{{ ELASTICMERGED.retention_pct }}
|
||||
|
||||
{%- set log_size_limit = salt['pillar.get']('elasticsearch:log_size_limit') %}
|
||||
actions:
|
||||
1:
|
||||
action: delete_indices
|
||||
description: >-
|
||||
Delete indices when {{log_size_limit}}(GB) is exceeded.
|
||||
options:
|
||||
allow_ilm_indices: True
|
||||
ignore_empty_list: True
|
||||
disable_action: False
|
||||
filters:
|
||||
- filtertype: pattern
|
||||
kind: regex
|
||||
value: '^(logstash-.*|so-.*|.ds-logs-.*-so.*)$'
|
||||
- filtertype: pattern
|
||||
kind: regex
|
||||
value: '^(so-case.*)$'
|
||||
exclude: True
|
||||
- filtertype: space
|
||||
source: creation_date
|
||||
use_age: True
|
||||
disk_space: {{log_size_limit}}
|
||||
@@ -1,27 +0,0 @@
|
||||
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
|
||||
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
|
||||
# https://securityonion.net/license; you may not use this file except in compliance with the
|
||||
# Elastic License 2.0.
|
||||
|
||||
{%- set DELETE_DAYS = CURATORMERGED['so-beats'].delete %}
|
||||
actions:
|
||||
1:
|
||||
action: delete_indices
|
||||
description: >-
|
||||
Delete beats indices when older than {{ DELETE_DAYS }} days.
|
||||
options:
|
||||
ignore_empty_list: True
|
||||
disable_action: False
|
||||
filters:
|
||||
- filtertype: pattern
|
||||
kind: regex
|
||||
value: '^(logstash-beats.*|so-beats.*)$'
|
||||
- filtertype: age
|
||||
source: name
|
||||
direction: older
|
||||
timestring: '%Y.%m.%d'
|
||||
unit: days
|
||||
unit_count: {{ DELETE_DAYS }}
|
||||
exclude:
|
||||
|
||||
|
||||
@@ -1,27 +0,0 @@
|
||||
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
|
||||
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
|
||||
# https://securityonion.net/license; you may not use this file except in compliance with the
|
||||
# Elastic License 2.0.
|
||||
|
||||
{%- set DELETE_DAYS = CURATORMERGED['so-elasticsearch'].delete %}
|
||||
actions:
|
||||
1:
|
||||
action: delete_indices
|
||||
description: >-
|
||||
Delete elasticsearch indices when older than {{ DELETE_DAYS }} days.
|
||||
options:
|
||||
ignore_empty_list: True
|
||||
disable_action: False
|
||||
filters:
|
||||
- filtertype: pattern
|
||||
kind: regex
|
||||
value: '^(logstash-elasticsearch.*|so-elasticsearch.*)$'
|
||||
- filtertype: age
|
||||
source: name
|
||||
direction: older
|
||||
timestring: '%Y.%m.%d'
|
||||
unit: days
|
||||
unit_count: {{ DELETE_DAYS }}
|
||||
exclude:
|
||||
|
||||
|
||||
@@ -1,28 +0,0 @@
|
||||
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
|
||||
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
|
||||
# https://securityonion.net/license; you may not use this file except in compliance with the
|
||||
# Elastic License 2.0.
|
||||
|
||||
|
||||
{%- set DELETE_DAYS = CURATORMERGED['so-firewall'].delete %}
|
||||
actions:
|
||||
1:
|
||||
action: delete_indices
|
||||
description: >-
|
||||
Delete firewall indices when older than {{ DELETE_DAYS }} days.
|
||||
options:
|
||||
ignore_empty_list: True
|
||||
disable_action: False
|
||||
filters:
|
||||
- filtertype: pattern
|
||||
kind: regex
|
||||
value: '^(logstash-firewall.*|so-firewall.*)$'
|
||||
- filtertype: age
|
||||
source: name
|
||||
direction: older
|
||||
timestring: '%Y.%m.%d'
|
||||
unit: days
|
||||
unit_count: {{ DELETE_DAYS }}
|
||||
exclude:
|
||||
|
||||
|
||||
@@ -1,28 +0,0 @@
|
||||
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
|
||||
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
|
||||
# https://securityonion.net/license; you may not use this file except in compliance with the
|
||||
# Elastic License 2.0.
|
||||
|
||||
|
||||
{%- set DELETE_DAYS = CURATORMERGED['so-ids'].delete %}
|
||||
actions:
|
||||
1:
|
||||
action: delete_indices
|
||||
description: >-
|
||||
Delete IDS indices when older than {{ DELETE_DAYS }} days.
|
||||
options:
|
||||
ignore_empty_list: True
|
||||
disable_action: False
|
||||
filters:
|
||||
- filtertype: pattern
|
||||
kind: regex
|
||||
value: '^(logstash-ids.*|so-ids.*)$'
|
||||
- filtertype: age
|
||||
source: name
|
||||
direction: older
|
||||
timestring: '%Y.%m.%d'
|
||||
unit: days
|
||||
unit_count: {{ DELETE_DAYS }}
|
||||
exclude:
|
||||
|
||||
|
||||
@@ -1,27 +0,0 @@
|
||||
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
|
||||
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
|
||||
# https://securityonion.net/license; you may not use this file except in compliance with the
|
||||
# Elastic License 2.0.
|
||||
|
||||
{%- set DELETE_DAYS = CURATORMERGED['so-import'].delete %}
|
||||
actions:
|
||||
1:
|
||||
action: delete_indices
|
||||
description: >-
|
||||
Delete import indices when older than {{ DELETE_DAYS }} days.
|
||||
options:
|
||||
ignore_empty_list: True
|
||||
disable_action: False
|
||||
filters:
|
||||
- filtertype: pattern
|
||||
kind: regex
|
||||
value: '^(logstash-import.*|so-import.*)$'
|
||||
- filtertype: age
|
||||
source: name
|
||||
direction: older
|
||||
timestring: '%Y.%m.%d'
|
||||
unit: days
|
||||
unit_count: {{ DELETE_DAYS }}
|
||||
exclude:
|
||||
|
||||
|
||||
@@ -1,27 +0,0 @@
|
||||
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
|
||||
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
|
||||
# https://securityonion.net/license; you may not use this file except in compliance with the
|
||||
# Elastic License 2.0.
|
||||
|
||||
{%- set DELETE_DAYS = CURATORMERGED['so-kibana'].delete %}
|
||||
actions:
|
||||
1:
|
||||
action: delete_indices
|
||||
description: >-
|
||||
Delete kibana indices when older than {{ DELETE_DAYS }} days.
|
||||
options:
|
||||
ignore_empty_list: True
|
||||
disable_action: False
|
||||
filters:
|
||||
- filtertype: pattern
|
||||
kind: regex
|
||||
value: '^(logstash-kibana.*|so-kibana.*)$'
|
||||
- filtertype: age
|
||||
source: name
|
||||
direction: older
|
||||
timestring: '%Y.%m.%d'
|
||||
unit: days
|
||||
unit_count: {{ DELETE_DAYS }}
|
||||
exclude:
|
||||
|
||||
|
||||
@@ -1,27 +0,0 @@
|
||||
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
|
||||
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
|
||||
# https://securityonion.net/license; you may not use this file except in compliance with the
|
||||
# Elastic License 2.0.
|
||||
|
||||
{%- set DELETE_DAYS = CURATORMERGED['so-kratos'].delete %}
|
||||
actions:
|
||||
1:
|
||||
action: delete_indices
|
||||
description: >-
|
||||
Delete kratos indices when older than {{ DELETE_DAYS }} days.
|
||||
options:
|
||||
ignore_empty_list: True
|
||||
disable_action: False
|
||||
filters:
|
||||
- filtertype: pattern
|
||||
kind: regex
|
||||
value: '^(logstash-kratos.*|so-kratos.*)$'
|
||||
- filtertype: age
|
||||
source: name
|
||||
direction: older
|
||||
timestring: '%Y.%m.%d'
|
||||
unit: days
|
||||
unit_count: {{ DELETE_DAYS }}
|
||||
exclude:
|
||||
|
||||
|
||||
@@ -1,27 +0,0 @@
|
||||
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
|
||||
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
|
||||
# https://securityonion.net/license; you may not use this file except in compliance with the
|
||||
# Elastic License 2.0.
|
||||
|
||||
{%- set DELETE_DAYS = CURATORMERGED['so-logstash'].delete %}
|
||||
actions:
|
||||
1:
|
||||
action: delete_indices
|
||||
description: >-
|
||||
Delete logstash indices when older than {{ DELETE_DAYS }} days.
|
||||
options:
|
||||
ignore_empty_list: True
|
||||
disable_action: False
|
||||
filters:
|
||||
- filtertype: pattern
|
||||
kind: regex
|
||||
value: '^(logstash-logstash.*|so-logstash.*)$'
|
||||
- filtertype: age
|
||||
source: name
|
||||
direction: older
|
||||
timestring: '%Y.%m.%d'
|
||||
unit: days
|
||||
unit_count: {{ DELETE_DAYS }}
|
||||
exclude:
|
||||
|
||||
|
||||
@@ -1,27 +0,0 @@
|
||||
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
|
||||
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
|
||||
# https://securityonion.net/license; you may not use this file except in compliance with the
|
||||
# Elastic License 2.0.
|
||||
|
||||
{%- set DELETE_DAYS = CURATORMERGED['so-netflow'].delete %}
|
||||
actions:
|
||||
1:
|
||||
action: delete_indices
|
||||
description: >-
|
||||
Delete netflow indices when older than {{ DELETE_DAYS }} days.
|
||||
options:
|
||||
ignore_empty_list: True
|
||||
disable_action: False
|
||||
filters:
|
||||
- filtertype: pattern
|
||||
kind: regex
|
||||
value: '^(logstash-netflow.*|so-netflow.*)$'
|
||||
- filtertype: age
|
||||
source: name
|
||||
direction: older
|
||||
timestring: '%Y.%m.%d'
|
||||
unit: days
|
||||
unit_count: {{ DELETE_DAYS }}
|
||||
exclude:
|
||||
|
||||
|
||||
@@ -1,27 +0,0 @@
|
||||
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
|
||||
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
|
||||
# https://securityonion.net/license; you may not use this file except in compliance with the
|
||||
# Elastic License 2.0.
|
||||
|
||||
{%- set DELETE_DAYS = CURATORMERGED['so-osquery'].delete %}
|
||||
actions:
|
||||
1:
|
||||
action: delete_indices
|
||||
description: >-
|
||||
Delete import indices when older than {{ DELETE_DAYS }} days.
|
||||
options:
|
||||
ignore_empty_list: True
|
||||
disable_action: False
|
||||
filters:
|
||||
- filtertype: pattern
|
||||
kind: regex
|
||||
value: '^(logstash-osquery.*|so-osquery.*)$'
|
||||
- filtertype: age
|
||||
source: name
|
||||
direction: older
|
||||
timestring: '%Y.%m.%d'
|
||||
unit: days
|
||||
unit_count: {{ DELETE_DAYS }}
|
||||
exclude:
|
||||
|
||||
|
||||
@@ -1,27 +0,0 @@
|
||||
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
|
||||
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
|
||||
# https://securityonion.net/license; you may not use this file except in compliance with the
|
||||
# Elastic License 2.0.
|
||||
|
||||
{%- set DELETE_DAYS = CURATORMERGED['so-ossec'].delete %}
|
||||
actions:
|
||||
1:
|
||||
action: delete_indices
|
||||
description: >-
|
||||
Delete ossec indices when older than {{ DELETE_DAYS }} days.
|
||||
options:
|
||||
ignore_empty_list: True
|
||||
disable_action: False
|
||||
filters:
|
||||
- filtertype: pattern
|
||||
kind: regex
|
||||
value: '^(logstash-ossec.*|so-ossec.*)$'
|
||||
- filtertype: age
|
||||
source: name
|
||||
direction: older
|
||||
timestring: '%Y.%m.%d'
|
||||
unit: days
|
||||
unit_count: {{ DELETE_DAYS }}
|
||||
exclude:
|
||||
|
||||
|
||||
@@ -1,27 +0,0 @@
|
||||
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
|
||||
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
|
||||
# https://securityonion.net/license; you may not use this file except in compliance with the
|
||||
# Elastic License 2.0.
|
||||
|
||||
{%- set DELETE_DAYS = CURATORMERGED['so-redis'].delete %}
|
||||
actions:
|
||||
1:
|
||||
action: delete_indices
|
||||
description: >-
|
||||
Delete redis indices when older than {{ DELETE_DAYS }} days.
|
||||
options:
|
||||
ignore_empty_list: True
|
||||
disable_action: False
|
||||
filters:
|
||||
- filtertype: pattern
|
||||
kind: regex
|
||||
value: '^(logstash-redis.*|so-redis.*)$'
|
||||
- filtertype: age
|
||||
source: name
|
||||
direction: older
|
||||
timestring: '%Y.%m.%d'
|
||||
unit: days
|
||||
unit_count: {{ DELETE_DAYS }}
|
||||
exclude:
|
||||
|
||||
|
||||
@@ -1,27 +0,0 @@
|
||||
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
|
||||
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
|
||||
# https://securityonion.net/license; you may not use this file except in compliance with the
|
||||
# Elastic License 2.0.
|
||||
|
||||
{%- set DELETE_DAYS = CURATORMERGED['so-strelka'].delete %}
|
||||
actions:
|
||||
1:
|
||||
action: delete_indices
|
||||
description: >-
|
||||
Delete Strelka indices when older than {{ DELETE_DAYS }} days.
|
||||
options:
|
||||
ignore_empty_list: True
|
||||
disable_action: False
|
||||
filters:
|
||||
- filtertype: pattern
|
||||
kind: regex
|
||||
value: '^(logstash-strelka.*|so-strelka.*)$'
|
||||
- filtertype: age
|
||||
source: name
|
||||
direction: older
|
||||
timestring: '%Y.%m.%d'
|
||||
unit: days
|
||||
unit_count: {{ DELETE_DAYS }}
|
||||
exclude:
|
||||
|
||||
|
||||
@@ -1,27 +0,0 @@
|
||||
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
|
||||
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
|
||||
# https://securityonion.net/license; you may not use this file except in compliance with the
|
||||
# Elastic License 2.0.
|
||||
|
||||
{%- set DELETE_DAYS = CURATORMERGED['so-syslog'].delete %}
|
||||
actions:
|
||||
1:
|
||||
action: delete_indices
|
||||
description: >-
|
||||
Delete syslog indices when older than {{ DELETE_DAYS }} days.
|
||||
options:
|
||||
ignore_empty_list: True
|
||||
disable_action: False
|
||||
filters:
|
||||
- filtertype: pattern
|
||||
kind: regex
|
||||
value: '^(logstash-syslog.*|so-syslog.*)$'
|
||||
- filtertype: age
|
||||
source: name
|
||||
direction: older
|
||||
timestring: '%Y.%m.%d'
|
||||
unit: days
|
||||
unit_count: {{ DELETE_DAYS }}
|
||||
exclude:
|
||||
|
||||
|
||||
@@ -1,27 +0,0 @@
|
||||
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
|
||||
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
|
||||
# https://securityonion.net/license; you may not use this file except in compliance with the
|
||||
# Elastic License 2.0.
|
||||
|
||||
{%- set DELETE_DAYS = CURATORMERGED['so-zeek'].delete %}
|
||||
actions:
|
||||
1:
|
||||
action: delete_indices
|
||||
description: >-
|
||||
Delete Zeek indices when older than {{ DELETE_DAYS }} days.
|
||||
options:
|
||||
ignore_empty_list: True
|
||||
disable_action: False
|
||||
filters:
|
||||
- filtertype: pattern
|
||||
kind: regex
|
||||
value: '^(logstash-zeek.*|so-zeek.*)$'
|
||||
- filtertype: age
|
||||
source: name
|
||||
direction: older
|
||||
timestring: '%Y.%m.%d'
|
||||
unit: days
|
||||
unit_count: {{ DELETE_DAYS }}
|
||||
exclude:
|
||||
|
||||
|
||||
@@ -1,40 +0,0 @@
|
||||
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
|
||||
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
|
||||
# https://securityonion.net/license; you may not use this file except in compliance with the
|
||||
# Elastic License 2.0.
|
||||
|
||||
{% from 'vars/globals.map.jinja' import GLOBALS %}
|
||||
{% if GLOBALS.role in ['so-searchnode', 'so-heavynode'] %}
|
||||
{%- set elasticsearch = GLOBALS.node_ip -%}
|
||||
{% elif GLOBALS.role in ['so-eval', 'so-managersearch', 'so-standalone', 'so-manager'] %}
|
||||
{%- set elasticsearch = GLOBALS.manager_ip -%}
|
||||
{%- endif %}
|
||||
{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %}
|
||||
{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %}
|
||||
|
||||
---
|
||||
# Remember, leave a key empty if there is no value. None will be a string,
|
||||
# not a Python "NoneType"
|
||||
elasticsearch:
|
||||
client:
|
||||
hosts:
|
||||
- https://{{elasticsearch}}:9200
|
||||
cloud_id:
|
||||
ca_certs:
|
||||
client_cert:
|
||||
client_key:
|
||||
verify_certs: False
|
||||
request_timeout: 30
|
||||
other_settings:
|
||||
api_key:
|
||||
id:
|
||||
api_key:
|
||||
master_only: False
|
||||
username: "{{ ES_USER }}"
|
||||
password: "{{ ES_PASS }}"
|
||||
|
||||
logging:
|
||||
loglevel: INFO
|
||||
logfile: '/var/log/curator/curator.log'
|
||||
logformat: default
|
||||
blacklist: ['elasticsearch', 'urllib3']
|
||||
@@ -1,13 +0,0 @@
|
||||
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
|
||||
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
|
||||
# https://securityonion.net/license; you may not use this file except in compliance with the
|
||||
# Elastic License 2.0.
|
||||
|
||||
{% from 'curator/map.jinja' import CURATORMERGED %}
|
||||
|
||||
include:
|
||||
{% if CURATORMERGED.enabled %}
|
||||
- curator.enabled
|
||||
{% else %}
|
||||
- curator.disabled
|
||||
{% endif %}
|
||||
@@ -1,7 +0,0 @@
|
||||
{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
|
||||
or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
|
||||
https://securityonion.net/license; you may not use this file except in compliance with the
|
||||
Elastic License 2.0. #}
|
||||
|
||||
{% import_yaml 'curator/defaults.yaml' as CURATORDEFAULTS %}
|
||||
{% set CURATORMERGED = salt['pillar.get']('curator', CURATORDEFAULTS.curator, merge=true) %}
|
||||
@@ -1,39 +0,0 @@
|
||||
curator:
|
||||
enabled:
|
||||
description: You can enable or disable Curator.
|
||||
helpLink: curator.html
|
||||
elasticsearch:
|
||||
index_settings:
|
||||
so-beats:
|
||||
delete: &delete
|
||||
description: Age, in days, when Curator deletes the index.
|
||||
helpLink: curator.html
|
||||
forcedType: int
|
||||
so-elasticsearch:
|
||||
delete: *delete
|
||||
so-firewall:
|
||||
delete: *delete
|
||||
so-ids:
|
||||
delete: *delete
|
||||
so-import:
|
||||
delete: *delete
|
||||
so-kratos:
|
||||
delete: *delete
|
||||
so-kibana:
|
||||
delete: *delete
|
||||
so-logstash:
|
||||
delete: *delete
|
||||
so-netflow:
|
||||
delete: *delete
|
||||
so-osquery:
|
||||
delete: *delete
|
||||
so-ossec:
|
||||
delete: *delete
|
||||
so-redis:
|
||||
delete: *delete
|
||||
so-strelka:
|
||||
delete: *delete
|
||||
so-syslog:
|
||||
delete: *delete
|
||||
so-zeek:
|
||||
delete: *delete
|
||||
@@ -1,21 +0,0 @@
|
||||
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
|
||||
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
|
||||
# https://securityonion.net/license; you may not use this file except in compliance with the
|
||||
# Elastic License 2.0.
|
||||
|
||||
{% from 'allowed_states.map.jinja' import allowed_states %}
|
||||
{% if sls.split('.')[0] in allowed_states %}
|
||||
|
||||
append_so-curator_so-status.conf:
|
||||
file.append:
|
||||
- name: /opt/so/conf/so-status/so-status.conf
|
||||
- text: so-curator
|
||||
- unless: grep -q so-curator /opt/so/conf/so-status/so-status.conf
|
||||
|
||||
{% else %}
|
||||
|
||||
{{sls}}_state_not_allowed:
|
||||
test.fail_without_changes:
|
||||
- name: {{sls}}_state_not_allowed
|
||||
|
||||
{% endif %}
|
||||
@@ -1,17 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
|
||||
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
|
||||
# https://securityonion.net/license; you may not use this file except in compliance with the
|
||||
# Elastic License 2.0.
|
||||
|
||||
# Avoid starting multiple instances
|
||||
APP=clusterdelete
|
||||
lf=/tmp/$APP-pidLockFile
|
||||
# create empty lock file if none exists
|
||||
cat /dev/null >> $lf
|
||||
read lastPID < $lf
|
||||
# if lastPID is not null and a process with that pid exists , exit
|
||||
[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit
|
||||
echo $$ > $lf
|
||||
|
||||
/usr/sbin/so-curator-cluster-delete-delete
|
||||
@@ -1,17 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
|
||||
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
|
||||
# https://securityonion.net/license; you may not use this file except in compliance with the
|
||||
# Elastic License 2.0.
|
||||
|
||||
|
||||
APP=delete
|
||||
lf=/tmp/$APP-pidLockFile
|
||||
# create empty lock file if none exists
|
||||
cat /dev/null >> $lf
|
||||
read lastPID < $lf
|
||||
# if lastPID is not null and a process with that pid exists , exit
|
||||
[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit
|
||||
echo $$ > $lf
|
||||
|
||||
docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/delete.yml > /dev/null 2>&1
|
||||
@@ -1,12 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
|
||||
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
|
||||
# https://securityonion.net/license; you may not use this file except in compliance with the
|
||||
# Elastic License 2.0.
|
||||
|
||||
|
||||
|
||||
. /usr/sbin/so-common
|
||||
|
||||
/usr/sbin/so-restart curator $1
|
||||
@@ -1,12 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
|
||||
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
|
||||
# https://securityonion.net/license; you may not use this file except in compliance with the
|
||||
# Elastic License 2.0.
|
||||
|
||||
|
||||
|
||||
. /usr/sbin/so-common
|
||||
|
||||
/usr/sbin/so-start curator $1
|
||||
@@ -1,12 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
|
||||
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
|
||||
# https://securityonion.net/license; you may not use this file except in compliance with the
|
||||
# Elastic License 2.0.
|
||||
|
||||
|
||||
|
||||
. /usr/sbin/so-common
|
||||
|
||||
/usr/sbin/so-stop curator $1
|
||||
@@ -1,67 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
|
||||
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
|
||||
# https://securityonion.net/license; you may not use this file except in compliance with the
|
||||
# Elastic License 2.0.
|
||||
|
||||
{% from 'vars/globals.map.jinja' import GLOBALS %}
|
||||
{% import_yaml 'elasticsearch/defaults.yaml' as ELASTICDEFAULTS %}
|
||||
{%- set ELASTICSEARCH_HOST = GLOBALS.node_ip -%}
|
||||
{%- set RETENTION = salt['pillar.get']('elasticsearch:retention', ELASTICDEFAULTS.elasticsearch.retention, merge=true) -%}
|
||||
|
||||
LOG="/opt/so/log/curator/so-curator-cluster-delete.log"
|
||||
ALERT_LOG="/opt/so/log/curator/alert.log"
|
||||
LOG_SIZE_LIMIT_GB=$(/usr/sbin/so-elasticsearch-cluster-space-total {{ RETENTION.retention_pct}})
|
||||
LOG_SIZE_LIMIT=$(( "$LOG_SIZE_LIMIT_GB" * 1000 * 1000 * 1000 ))
|
||||
ITERATION=0
|
||||
MAX_ITERATIONS=10
|
||||
|
||||
overlimit() {
|
||||
[[ $(/usr/sbin/so-elasticsearch-cluster-space-used) -gt ${LOG_SIZE_LIMIT} ]]
|
||||
}
|
||||
|
||||
###########################
|
||||
# Check for 2 conditions: #
|
||||
###########################
|
||||
# 1. Check if Elasticsearch indices are using more disk space than LOG_SIZE_LIMIT
|
||||
# 2. Check if the maximum number of iterations - MAX_ITERATIONS - has been exceeded. If so, exit.
|
||||
# Closed indices will be deleted first. If we are able to bring disk space under LOG_SIZE_LIMIT, or the number of iterations has exceeded the maximum allowed number of iterations, we will break out of the loop.
|
||||
|
||||
while overlimit && [[ $ITERATION -lt $MAX_ITERATIONS ]]; do
|
||||
|
||||
# If we can't query Elasticsearch, then immediately return false.
|
||||
/usr/sbin/so-elasticsearch-query _cat/indices?h=index,status > /dev/null 2>&1
|
||||
[ $? -eq 1 ] && echo "$(date) - Could not query Elasticsearch." >> ${LOG} && exit
|
||||
|
||||
# We iterate through the closed and open indices
|
||||
CLOSED_INDICES=$(/usr/sbin/so-elasticsearch-query _cat/indices?h=index,status | grep 'close$' | awk '{print $1}' | grep -vE "playbook|so-case" | grep -E "(logstash-|so-|.ds-logs-)" | sort -t- -k3)
|
||||
OPEN_INDICES=$(/usr/sbin/so-elasticsearch-query _cat/indices?h=index,status | grep 'open$' | awk '{print $1}' | grep -vE "playbook|so-case" | grep -E "(logstash-|so-|.ds-logs-)" | sort -t- -k3)
|
||||
|
||||
for INDEX in ${CLOSED_INDICES} ${OPEN_INDICES}; do
|
||||
# Now that we've sorted the indices from oldest to newest, we need to check each index to see if it is assigned as the current write index for a data stream
|
||||
# To do so, we need to identify to which data stream this index is associated
|
||||
# We extract the data stream name using the pattern below
|
||||
DATASTREAM_PATTERN="logs-[a-zA-Z_.]+-[a-zA-Z_.]+"
|
||||
DATASTREAM=$(echo "${INDEX}" | grep -oE "$DATASTREAM_PATTERN")
|
||||
# We look up the data stream, and determine the write index. If there is only one backing index, we delete the entire data stream
|
||||
BACKING_INDICES=$(/usr/sbin/so-elasticsearch-query _data_stream/${DATASTREAM} | jq -r '.data_streams[0].indices | length')
|
||||
if [ "$BACKING_INDICES" -gt 1 ]; then
|
||||
CURRENT_WRITE_INDEX=$(/usr/sbin/so-elasticsearch-query _data_stream/$DATASTREAM | jq -r .data_streams[0].indices[-1].index_name)
|
||||
# We make sure we are not trying to delete a write index
|
||||
if [ "${INDEX}" != "${CURRENT_WRITE_INDEX}" ]; then
|
||||
# This should not be a write index, so we should be allowed to delete it
|
||||
printf "\n$(date) - Used disk space exceeds LOG_SIZE_LIMIT (${LOG_SIZE_LIMIT_GB} GB) - Deleting ${INDEX} index...\n" >> ${LOG}
|
||||
/usr/sbin/so-elasticsearch-query ${INDEX} -XDELETE >> ${LOG} 2>&1
|
||||
fi
|
||||
fi
|
||||
if ! overlimit ; then
|
||||
exit
|
||||
fi
|
||||
((ITERATION++))
|
||||
done
|
||||
if [[ $ITERATION -ge $MAX_ITERATIONS ]]; then
|
||||
alert_id=$(uuidgen)
|
||||
printf "\n$(date) -> Maximum iteration limit reached ($MAX_ITERATIONS). Unable to bring disk below threshold. Writing alert ($alert_id) to ${ALERT_LOG}\n" >> ${LOG}
|
||||
printf "\n$(date),$alert_id,Maximum iteration limit reached ($MAX_ITERATIONS). Unable to bring disk below threshold.\n" >> ${ALERT_LOG}
|
||||
fi
|
||||
done
|
||||
Reference in New Issue
Block a user