CSEC_PUBLIC/securityonion
1
0
Fork 0
mirror of https://github.com/Security-Onion-Solutions/securityonion.git synced 2025-12-06 09:12:45 +01:00
Code Issues Packages Projects Releases Wiki Activity

Remove Curator

Browse Source
This commit is contained in:
Wes
2023-12-08 19:37:06 +00:00
parent be8ed1e1d8
commit d203aec44a
47 changed files with 40 additions and 910 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
pillar/top.sls
View File

@@ -61,8 +61,6 @@ base:
- elastalert.adv_elastalert
- backup.soc_backup
- backup.adv_backup
- curator.soc_curator
- curator.adv_curator
- soctopus.soc_soctopus
- soctopus.adv_soctopus
- minions.{{ grains.id }}
@@ -113,8 +111,6 @@ base:
- kibana.adv_kibana
- strelka.soc_strelka
- strelka.adv_strelka
- curator.soc_curator
- curator.adv_curator
- kratos.soc_kratos
- kratos.adv_kratos
- redis.soc_redis
@@ -172,8 +168,6 @@ base:
- kibana.adv_kibana
- strelka.soc_strelka
- strelka.adv_strelka
- curator.soc_curator
- curator.adv_curator
- backup.soc_backup
- backup.adv_backup
- zeek.soc_zeek
@@ -194,8 +188,6 @@ base:
- logstash.adv_logstash
- elasticsearch.soc_elasticsearch
- elasticsearch.adv_elasticsearch
- curator.soc_curator
- curator.adv_curator
- redis.soc_redis
- redis.adv_redis
- zeek.soc_zeek
@@ -268,8 +260,6 @@ base:
- soctopus.adv_soctopus
- kibana.soc_kibana
- kibana.adv_kibana
- curator.soc_curator
- curator.adv_curator
- backup.soc_backup
- backup.adv_backup
- kratos.soc_kratos

4
salt/allowed_states.map.jinja
View File

@@ -219,10 +219,6 @@
{% do allowed_states.append('kibana.secrets') %}
{% endif %}
{% if grains.role in ['so-eval', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-manager'] %}
{% do allowed_states.append('curator') %}
{% endif %}
{% if grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-managersearch'] %}
{% do allowed_states.append('elastalert') %}
{% endif %}

1
salt/common/tools/sbin/so-image-common
View File

@@ -42,7 +42,6 @@ container_list() {
)
elif [ $MANAGERCHECK != 'so-helix' ]; then
TRUSTED_CONTAINERS=(
"so-curator"
"so-elastalert"
"so-elastic-agent"
"so-elastic-agent-builder"

81
salt/curator/config.sls
View File

@@ -1,81 +0,0 @@
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{% from 'allowed_states.map.jinja' import allowed_states %}
{% if sls.split('.')[0] in allowed_states %}
{% from "curator/map.jinja" import CURATORMERGED %}
# Create the group
curatorgroup:
group.present:
- name: curator
- gid: 934
# Add user
curator:
user.present:
- uid: 934
- gid: 934
- home: /opt/so/conf/curator
- createhome: False
# Create the log directory
curlogdir:
file.directory:
- name: /opt/so/log/curator
- user: 934
- group: 939
curactiondir:
file.directory:
- name: /opt/so/conf/curator/action
- user: 934
- group: 939
- makedirs: True
actionconfs:
file.recurse:
- name: /opt/so/conf/curator/action
- source: salt://curator/files/action
- user: 934
- group: 939
- template: jinja
- defaults:
CURATORMERGED: {{ CURATORMERGED.elasticsearch.index_settings }}
curconf:
file.managed:
- name: /opt/so/conf/curator/curator.yml
- source: salt://curator/files/curator.yml
- user: 934
- group: 939
- mode: 660
- template: jinja
- show_changes: False
curator_sbin:
file.recurse:
- name: /usr/sbin
- source: salt://curator/tools/sbin
- user: 934
- group: 939
- file_mode: 755
curator_sbin_jinja:
file.recurse:
- name: /usr/sbin
- source: salt://curator/tools/sbin_jinja
- user: 934
- group: 939
- file_mode: 755
- template: jinja
{% else %}
{{sls}}_state_not_allowed:
test.fail_without_changes:
- name: {{sls}}_state_not_allowed
{% endif %}

34
salt/curator/defaults.yaml
View File

@@ -1,34 +0,0 @@
curator:
enabled: False
elasticsearch:
index_settings:
so-beats:
delete: 365
so-elasticsearch:
delete: 365
so-firewall:
delete: 365
so-ids:
delete: 365
so-import:
delete: 73001
so-kratos:
delete: 365
so-kibana:
delete: 365
so-logstash:
delete: 365
so-netflow:
delete: 365
so-osquery:
delete: 365
so-ossec:
delete: 365
so-redis:
delete: 365
so-strelka:
delete: 365
so-syslog:
delete: 365
so-zeek:
delete: 365

35
salt/curator/disabled.sls
View File

@@ -1,35 +0,0 @@
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{% from 'allowed_states.map.jinja' import allowed_states %}
{% if sls.split('.')[0] in allowed_states %}
include:
- curator.sostatus
so-curator:
docker_container.absent:
- force: True
so-curator_so-status.disabled:
file.comment:
- name: /opt/so/conf/so-status/so-status.conf
- regex: ^so-curator$
so-curator-cluster-close:
cron.absent:
- identifier: so-curator-cluster-close
so-curator-cluster-delete:
cron.absent:
- identifier: so-curator-cluster-delete
{% else %}
{{sls}}_state_not_allowed:
test.fail_without_changes:
- name: {{sls}}_state_not_allowed
{% endif %}

81
salt/curator/enabled.sls
View File

@@ -1,81 +0,0 @@
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{% from 'allowed_states.map.jinja' import allowed_states %}
{% if sls.split('.')[0] in allowed_states %}
{% from 'vars/globals.map.jinja' import GLOBALS %}
{% from 'docker/docker.map.jinja' import DOCKER %}
include:
- curator.config
- curator.sostatus
so-curator:
docker_container.running:
- image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-curator:{{ GLOBALS.so_version }}
- start: True
- hostname: curator
- name: so-curator
- user: curator
- networks:
- sobridge:
- ipv4_address: {{ DOCKER.containers['so-curator'].ip }}
- interactive: True
- tty: True
- binds:
- /opt/so/conf/curator/curator.yml:/etc/curator/config/curator.yml:ro
- /opt/so/conf/curator/action/:/etc/curator/action:ro
- /opt/so/log/curator:/var/log/curator:rw
{% if DOCKER.containers['so-curator'].custom_bind_mounts %}
{% for BIND in DOCKER.containers['so-curator'].custom_bind_mounts %}
- {{ BIND }}
{% endfor %}
{% endif %}
{% if DOCKER.containers['so-curator'].extra_hosts %}
- extra_hosts:
{% for XTRAHOST in DOCKER.containers['so-curator'].extra_hosts %}
- {{ XTRAHOST }}
{% endfor %}
{% endif %}
{% if DOCKER.containers['so-curator'].extra_env %}
- environment:
{% for XTRAENV in DOCKER.containers['so-curator'].extra_env %}
- {{ XTRAENV }}
{% endfor %}
{% endif %}
- require:
- file: actionconfs
- file: curconf
- file: curlogdir
- watch:
- file: curconf
delete_so-curator_so-status.disabled:
file.uncomment:
- name: /opt/so/conf/so-status/so-status.conf
- regex: ^so-curator$
so-curator-cluster-close:
cron.absent:
- identifier: so-curator-cluster-close
so-curator-cluster-delete:
cron.present:
- name: /usr/sbin/so-curator-cluster-delete > /opt/so/log/curator/cron-cluster-delete.log 2>&1
- identifier: so-curator-cluster-delete
- user: root
- minute: '*/5'
- hour: '*'
- daymonth: '*'
- month: '*'
- dayweek: '*'
{% else %}
{{sls}}_state_not_allowed:
test.fail_without_changes:
- name: {{sls}}_state_not_allowed
{% endif %}

32
salt/curator/files/action/delete.yml
View File

@@ -1,32 +0,0 @@
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{% import_yaml 'elasticsearch/defaults.yaml' as ELASTICDEFAULTS %}
{% set ELASTICMERGED = salt['pillar.get']('elasticsearch:retention', ELASTICDEFAULTS.elasticsearch.retention, merge=true) %}
{{ ELASTICMERGED.retention_pct }}
{%- set log_size_limit = salt['pillar.get']('elasticsearch:log_size_limit') %}
actions:
1:
action: delete_indices
description: >-
Delete indices when {{log_size_limit}}(GB) is exceeded.
options:
allow_ilm_indices: True
ignore_empty_list: True
disable_action: False
filters:
- filtertype: pattern
kind: regex
value: '^(logstash-.*|so-.*|.ds-logs-.*-so.*)$'
- filtertype: pattern
kind: regex
value: '^(so-case.*)$'
exclude: True
- filtertype: space
source: creation_date
use_age: True
disk_space: {{log_size_limit}}

27
salt/curator/files/action/so-beats-delete.yml
View File

@@ -1,27 +0,0 @@
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{%- set DELETE_DAYS = CURATORMERGED['so-beats'].delete %}
actions:
1:
action: delete_indices
description: >-
Delete beats indices when older than {{ DELETE_DAYS }} days.
options:
ignore_empty_list: True
disable_action: False
filters:
- filtertype: pattern
kind: regex
value: '^(logstash-beats.*|so-beats.*)$'
- filtertype: age
source: name
direction: older
timestring: '%Y.%m.%d'
unit: days
unit_count: {{ DELETE_DAYS }}
exclude:

27
salt/curator/files/action/so-elasticsearch-delete.yml
View File

@@ -1,27 +0,0 @@
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{%- set DELETE_DAYS = CURATORMERGED['so-elasticsearch'].delete %}
actions:
1:
action: delete_indices
description: >-
Delete elasticsearch indices when older than {{ DELETE_DAYS }} days.
options:
ignore_empty_list: True
disable_action: False
filters:
- filtertype: pattern
kind: regex
value: '^(logstash-elasticsearch.*|so-elasticsearch.*)$'
- filtertype: age
source: name
direction: older
timestring: '%Y.%m.%d'
unit: days
unit_count: {{ DELETE_DAYS }}
exclude:

28
salt/curator/files/action/so-firewall-delete.yml
View File

@@ -1,28 +0,0 @@
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{%- set DELETE_DAYS = CURATORMERGED['so-firewall'].delete %}
actions:
1:
action: delete_indices
description: >-
Delete firewall indices when older than {{ DELETE_DAYS }} days.
options:
ignore_empty_list: True
disable_action: False
filters:
- filtertype: pattern
kind: regex
value: '^(logstash-firewall.*|so-firewall.*)$'
- filtertype: age
source: name
direction: older
timestring: '%Y.%m.%d'
unit: days
unit_count: {{ DELETE_DAYS }}
exclude:

28
salt/curator/files/action/so-ids-delete.yml
View File

@@ -1,28 +0,0 @@
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{%- set DELETE_DAYS = CURATORMERGED['so-ids'].delete %}
actions:
1:
action: delete_indices
description: >-
Delete IDS indices when older than {{ DELETE_DAYS }} days.
options:
ignore_empty_list: True
disable_action: False
filters:
- filtertype: pattern
kind: regex
value: '^(logstash-ids.*|so-ids.*)$'
- filtertype: age
source: name
direction: older
timestring: '%Y.%m.%d'
unit: days
unit_count: {{ DELETE_DAYS }}
exclude:

27
salt/curator/files/action/so-import-delete.yml
View File

@@ -1,27 +0,0 @@
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{%- set DELETE_DAYS = CURATORMERGED['so-import'].delete %}
actions:
1:
action: delete_indices
description: >-
Delete import indices when older than {{ DELETE_DAYS }} days.
options:
ignore_empty_list: True
disable_action: False
filters:
- filtertype: pattern
kind: regex
value: '^(logstash-import.*|so-import.*)$'
- filtertype: age
source: name
direction: older
timestring: '%Y.%m.%d'
unit: days
unit_count: {{ DELETE_DAYS }}
exclude:

27
salt/curator/files/action/so-kibana-delete.yml
View File

@@ -1,27 +0,0 @@
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{%- set DELETE_DAYS = CURATORMERGED['so-kibana'].delete %}
actions:
1:
action: delete_indices
description: >-
Delete kibana indices when older than {{ DELETE_DAYS }} days.
options:
ignore_empty_list: True
disable_action: False
filters:
- filtertype: pattern
kind: regex
value: '^(logstash-kibana.*|so-kibana.*)$'
- filtertype: age
source: name
direction: older
timestring: '%Y.%m.%d'
unit: days
unit_count: {{ DELETE_DAYS }}
exclude:

27
salt/curator/files/action/so-kratos-delete.yml
View File

@@ -1,27 +0,0 @@
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{%- set DELETE_DAYS = CURATORMERGED['so-kratos'].delete %}
actions:
1:
action: delete_indices
description: >-
Delete kratos indices when older than {{ DELETE_DAYS }} days.
options:
ignore_empty_list: True
disable_action: False
filters:
- filtertype: pattern
kind: regex
value: '^(logstash-kratos.*|so-kratos.*)$'
- filtertype: age
source: name
direction: older
timestring: '%Y.%m.%d'
unit: days
unit_count: {{ DELETE_DAYS }}
exclude:

27
salt/curator/files/action/so-logstash-delete.yml
View File

@@ -1,27 +0,0 @@
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{%- set DELETE_DAYS = CURATORMERGED['so-logstash'].delete %}
actions:
1:
action: delete_indices
description: >-
Delete logstash indices when older than {{ DELETE_DAYS }} days.
options:
ignore_empty_list: True
disable_action: False
filters:
- filtertype: pattern
kind: regex
value: '^(logstash-logstash.*|so-logstash.*)$'
- filtertype: age
source: name
direction: older
timestring: '%Y.%m.%d'
unit: days
unit_count: {{ DELETE_DAYS }}
exclude:

27
salt/curator/files/action/so-netflow-delete.yml
View File

@@ -1,27 +0,0 @@
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{%- set DELETE_DAYS = CURATORMERGED['so-netflow'].delete %}
actions:
1:
action: delete_indices
description: >-
Delete netflow indices when older than {{ DELETE_DAYS }} days.
options:
ignore_empty_list: True
disable_action: False
filters:
- filtertype: pattern
kind: regex
value: '^(logstash-netflow.*|so-netflow.*)$'
- filtertype: age
source: name
direction: older
timestring: '%Y.%m.%d'
unit: days
unit_count: {{ DELETE_DAYS }}
exclude:

27
salt/curator/files/action/so-osquery-delete.yml
View File

@@ -1,27 +0,0 @@
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{%- set DELETE_DAYS = CURATORMERGED['so-osquery'].delete %}
actions:
1:
action: delete_indices
description: >-
Delete import indices when older than {{ DELETE_DAYS }} days.
options:
ignore_empty_list: True
disable_action: False
filters:
- filtertype: pattern
kind: regex
value: '^(logstash-osquery.*|so-osquery.*)$'
- filtertype: age
source: name
direction: older
timestring: '%Y.%m.%d'
unit: days
unit_count: {{ DELETE_DAYS }}
exclude:

27
salt/curator/files/action/so-ossec-delete.yml
View File

@@ -1,27 +0,0 @@
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{%- set DELETE_DAYS = CURATORMERGED['so-ossec'].delete %}
actions:
1:
action: delete_indices
description: >-
Delete ossec indices when older than {{ DELETE_DAYS }} days.
options:
ignore_empty_list: True
disable_action: False
filters:
- filtertype: pattern
kind: regex
value: '^(logstash-ossec.*|so-ossec.*)$'
- filtertype: age
source: name
direction: older
timestring: '%Y.%m.%d'
unit: days
unit_count: {{ DELETE_DAYS }}
exclude:

27
salt/curator/files/action/so-redis-delete.yml
View File

@@ -1,27 +0,0 @@
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{%- set DELETE_DAYS = CURATORMERGED['so-redis'].delete %}
actions:
1:
action: delete_indices
description: >-
Delete redis indices when older than {{ DELETE_DAYS }} days.
options:
ignore_empty_list: True
disable_action: False
filters:
- filtertype: pattern
kind: regex
value: '^(logstash-redis.*|so-redis.*)$'
- filtertype: age
source: name
direction: older
timestring: '%Y.%m.%d'
unit: days
unit_count: {{ DELETE_DAYS }}
exclude:

27
salt/curator/files/action/so-strelka-delete.yml
View File

@@ -1,27 +0,0 @@
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{%- set DELETE_DAYS = CURATORMERGED['so-strelka'].delete %}
actions:
1:
action: delete_indices
description: >-
Delete Strelka indices when older than {{ DELETE_DAYS }} days.
options:
ignore_empty_list: True
disable_action: False
filters:
- filtertype: pattern
kind: regex
value: '^(logstash-strelka.*|so-strelka.*)$'
- filtertype: age
source: name
direction: older
timestring: '%Y.%m.%d'
unit: days
unit_count: {{ DELETE_DAYS }}
exclude:

27
salt/curator/files/action/so-syslog-delete.yml
View File

@@ -1,27 +0,0 @@
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{%- set DELETE_DAYS = CURATORMERGED['so-syslog'].delete %}
actions:
1:
action: delete_indices
description: >-
Delete syslog indices when older than {{ DELETE_DAYS }} days.
options:
ignore_empty_list: True
disable_action: False
filters:
- filtertype: pattern
kind: regex
value: '^(logstash-syslog.*|so-syslog.*)$'
- filtertype: age
source: name
direction: older
timestring: '%Y.%m.%d'
unit: days
unit_count: {{ DELETE_DAYS }}
exclude:

27
salt/curator/files/action/so-zeek-delete.yml
View File

@@ -1,27 +0,0 @@
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{%- set DELETE_DAYS = CURATORMERGED['so-zeek'].delete %}
actions:
1:
action: delete_indices
description: >-
Delete Zeek indices when older than {{ DELETE_DAYS }} days.
options:
ignore_empty_list: True
disable_action: False
filters:
- filtertype: pattern
kind: regex
value: '^(logstash-zeek.*|so-zeek.*)$'
- filtertype: age
source: name
direction: older
timestring: '%Y.%m.%d'
unit: days
unit_count: {{ DELETE_DAYS }}
exclude:

40
salt/curator/files/curator.yml
View File

@@ -1,40 +0,0 @@
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{% from 'vars/globals.map.jinja' import GLOBALS %}
{% if GLOBALS.role in ['so-searchnode', 'so-heavynode'] %}
{%- set elasticsearch = GLOBALS.node_ip -%}
{% elif GLOBALS.role in ['so-eval', 'so-managersearch', 'so-standalone', 'so-manager'] %}
{%- set elasticsearch = GLOBALS.manager_ip -%}
{%- endif %}
{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %}
{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %}
---
# Remember, leave a key empty if there is no value. None will be a string,
# not a Python "NoneType"
elasticsearch:
client:
hosts:
- https://{{elasticsearch}}:9200
cloud_id:
ca_certs:
client_cert:
client_key:
verify_certs: False
request_timeout: 30
other_settings:
api_key:
id:
api_key:
master_only: False
username: "{{ ES_USER }}"
password: "{{ ES_PASS }}"
logging:
loglevel: INFO
logfile: '/var/log/curator/curator.log'
logformat: default
blacklist: ['elasticsearch', 'urllib3']

13
salt/curator/init.sls
View File

@@ -1,13 +0,0 @@
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{% from 'curator/map.jinja' import CURATORMERGED %}
include:
{% if CURATORMERGED.enabled %}
- curator.enabled
{% else %}
- curator.disabled
{% endif %}

7
salt/curator/map.jinja
View File

@@ -1,7 +0,0 @@
{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
https://securityonion.net/license; you may not use this file except in compliance with the
Elastic License 2.0. #}
{% import_yaml 'curator/defaults.yaml' as CURATORDEFAULTS %}
{% set CURATORMERGED = salt['pillar.get']('curator', CURATORDEFAULTS.curator, merge=true) %}

39
salt/curator/soc_curator.yaml
View File

@@ -1,39 +0,0 @@
curator:
enabled:
description: You can enable or disable Curator.
helpLink: curator.html
elasticsearch:
index_settings:
so-beats:
delete: &delete
description: Age, in days, when Curator deletes the index.
helpLink: curator.html
forcedType: int
so-elasticsearch:
delete: *delete
so-firewall:
delete: *delete
so-ids:
delete: *delete
so-import:
delete: *delete
so-kratos:
delete: *delete
so-kibana:
delete: *delete
so-logstash:
delete: *delete
so-netflow:
delete: *delete
so-osquery:
delete: *delete
so-ossec:
delete: *delete
so-redis:
delete: *delete
so-strelka:
delete: *delete
so-syslog:
delete: *delete
so-zeek:
delete: *delete

21
salt/curator/sostatus.sls
View File

@@ -1,21 +0,0 @@
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{% from 'allowed_states.map.jinja' import allowed_states %}
{% if sls.split('.')[0] in allowed_states %}
append_so-curator_so-status.conf:
file.append:
- name: /opt/so/conf/so-status/so-status.conf
- text: so-curator
- unless: grep -q so-curator /opt/so/conf/so-status/so-status.conf
{% else %}
{{sls}}_state_not_allowed:
test.fail_without_changes:
- name: {{sls}}_state_not_allowed
{% endif %}

17
salt/curator/tools/sbin/so-curator-delete
View File

@@ -1,17 +0,0 @@
#!/bin/bash
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
APP=delete
lf=/tmp/$APP-pidLockFile
# create empty lock file if none exists
cat /dev/null >> $lf
read lastPID < $lf
# if lastPID is not null and a process with that pid exists , exit
[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit
echo $$ > $lf
docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/delete.yml > /dev/null 2>&1

12
salt/curator/tools/sbin/so-curator-restart
View File

@@ -1,12 +0,0 @@
#!/bin/bash
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
. /usr/sbin/so-common
/usr/sbin/so-restart curator $1

12
salt/curator/tools/sbin/so-curator-start
View File

@@ -1,12 +0,0 @@
#!/bin/bash
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
. /usr/sbin/so-common
/usr/sbin/so-start curator $1

12
salt/curator/tools/sbin/so-curator-stop
View File

@@ -1,12 +0,0 @@
#!/bin/bash
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
. /usr/sbin/so-common
/usr/sbin/so-stop curator $1

5
salt/docker/defaults.yaml
View File

@@ -159,11 +159,6 @@ docker:
custom_bind_mounts: []
extra_hosts: []
extra_env: []
'so-curator':
final_octet: 43
custom_bind_mounts: []
extra_hosts: []
extra_env: []
'so-elastic-fleet-package-registry':
final_octet: 44
port_bindings:

1
salt/docker/soc_docker.yaml
View File

@@ -60,7 +60,6 @@ docker:
so-strelka-gatekeeper: *dockerOptions
so-strelka-coordinator: *dockerOptions
so-elastalert: *dockerOptions
so-curator: *dockerOptions
so-elastic-fleet-package-registry: *dockerOptions
so-idh: *dockerOptions
so-elastic-agent: *dockerOptions

4
salt/elasticsearch/disabled.sls
View File

@@ -18,6 +18,10 @@ so-elasticsearch_so-status.disabled:
- name: /opt/so/conf/so-status/so-status.conf
- regex: ^so-elasticsearch$
so-elasticsearch-indices-delete:
cron.absent:
- identifier: so-elasticsearch-indices-delete
{% else %}
{{sls}}_state_not_allowed:

20
salt/elasticsearch/enabled.sls
View File

@@ -195,6 +195,26 @@ so-elasticsearch-roles-load:
- require:
- docker_container: so-elasticsearch
- file: elasticsearch_sbin_jinja
{% if grains.role in ['so-eval', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-manager'] %}
so-curator-cluster-close:
cron.absent:
- identifier: so-curator-cluster-close
so-curator-cluster-delete:
cron.absent:
- identifier: so-curator-cluster-delete
so-elasticsearch-indices-delete:
cron.present:
- name: /usr/sbin/so-elasticsearch-indices-delete > /opt/so/log/curator/cron-elasticsearch-indices-delete.log 2>&1
- identifier: so-elasticsearch-indices-delete
- user: root
- minute: '*/5'
- hour: '*'
- daymonth: '*'
- month: '*'
- dayweek: '*'
{% endif %}
{% endif %}
{% else %}

2
salt/curator/tools/sbin/so-curator-cluster-delete → salt/elasticsearch/tools/sbin/so-elasticsearch-indices-delete
View File

@@ -14,4 +14,4 @@ read lastPID < $lf
[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit
echo $$ > $lf
/usr/sbin/so-curator-cluster-delete-delete
/usr/sbin/so-elasticsearch-indices-delete-delete

4
salt/curator/tools/sbin_jinja/so-curator-cluster-delete-delete → salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-indices-delete-delete
View File

@@ -9,8 +9,8 @@
{%- set ELASTICSEARCH_HOST = GLOBALS.node_ip -%}
{%- set RETENTION = salt['pillar.get']('elasticsearch:retention', ELASTICDEFAULTS.elasticsearch.retention, merge=true) -%}
LOG="/opt/so/log/curator/so-curator-cluster-delete.log"
ALERT_LOG="/opt/so/log/curator/alert.log"
LOG="/opt/so/log/elasticsearch/so-elasticsearch-indices-delete.log"
ALERT_LOG="/opt/so/log/elasticsearch/indices-delete-alert.log"
LOG_SIZE_LIMIT_GB=$(/usr/sbin/so-elasticsearch-cluster-space-total {{ RETENTION.retention_pct}})
LOG_SIZE_LIMIT=$(( "$LOG_SIZE_LIMIT_GB" * 1000 * 1000 * 1000 ))
ITERATION=0

3
salt/firewall/containers.map.jinja
View File

@@ -2,7 +2,6 @@
{% if GLOBALS.role == 'so-eval' %}
{% set NODE_CONTAINERS = [
'so-curator',
'so-dockerregistry',
'so-elasticsearch',
'so-elastic-fleet',
@@ -25,7 +24,6 @@
{% elif GLOBALS.role == 'so-manager' or GLOBALS.role == 'so-standalone' or GLOBALS.role == 'so-managersearch' %}
{% set NODE_CONTAINERS = [
'so-curator',
'so-dockerregistry',
'so-elasticsearch',
'so-elastic-fleet',
@@ -56,7 +54,6 @@
{% elif GLOBALS.role == 'so-heavynode' %}
{% set NODE_CONTAINERS = [
'so-curator',
'so-elasticsearch',
'so-elastic-agent',
'so-logstash',

2
salt/logrotate/defaults.yaml
View File

@@ -80,7 +80,7 @@ logrotate:
- extension .log
- dateext
- dateyesterday
/opt/so/log/curator/*_x_log:
/opt/so/log/elasticsearch/*indices-delete*_x_log:
- daily
- rotate 14
- missingok

14
salt/logrotate/soc_logrotate.yaml
View File

@@ -56,13 +56,6 @@ logrotate:
multiline: True
global: True
forcedType: "[]string"
"/opt/so/log/curator/*_x_log":
description: List of logrotate options for this file.
title: /opt/so/log/curator/*.log
advanced: True
multiline: True
global: True
forcedType: "[]string"
"/opt/so/log/elasticfleet/*_x_log":
description: List of logrotate options for this file.
title: /opt/so/log/elasticfleet/*.log
@@ -77,6 +70,13 @@ logrotate:
multiline: True
global: True
forcedType: "[]string"
"/opt/so/log/elasticsearch/*indices-delete*_x_log":
description: List of logrotate options for this file.
title: /opt/so/log/elasticsearch/*indices-delete.log
advanced: True
multiline: True
global: True
forcedType: "[]string"
"/opt/so/log/suricata/*_x_log":
description: List of logrotate options for this file.
title: /opt/so/log/suricata/*.log

2
salt/manager/tools/sbin/so-elastic-auth-password-reset
View File

@@ -97,8 +97,6 @@ function soUserSync() {
salt -C 'G@role:so-standalone or G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managersearch or G@role:so-searchnode or G@role:so-heavynode' state.apply logstash queue=True
printf "\nApplying kibana state to the appropriate nodes.\n\n"
salt -C 'G@role:so-standalone or G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managersearch' state.apply kibana queue=True
printf "\nApplying curator state to the appropriate nodes.\n\n"
salt -C 'G@role:so-standalone or G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managersearch or G@role:so-searchnode or G@role:so-heavynode' state.apply curator queue=True
}
function highstateManager() {

12
salt/manager/tools/sbin/so-minion
View File

@@ -302,13 +302,6 @@ function add_strelka_to_minion() {
" " >> $PILLARFILE
}
function add_curator_to_minion() {
printf '%s\n'\
"curator:"\
" enabled: True"\
" " >> $PILLARFILE
}
function add_telegraf_to_minion() {
printf '%s\n'\
"telegraf:"\
@@ -435,7 +428,6 @@ function createEVAL() {
add_playbook_to_minion
add_elastalert_to_minion
add_kibana_to_minion
add_curator_to_minion
add_telegraf_to_minion
add_influxdb_to_minion
add_nginx_to_minion
@@ -458,7 +450,6 @@ function createSTANDALONE() {
add_elastalert_to_minion
add_kibana_to_minion
add_redis_to_minion
add_curator_to_minion
add_telegraf_to_minion
add_influxdb_to_minion
add_nginx_to_minion
@@ -478,7 +469,6 @@ function createMANAGER() {
add_elastalert_to_minion
add_kibana_to_minion
add_redis_to_minion
add_curator_to_minion
add_telegraf_to_minion
add_influxdb_to_minion
add_nginx_to_minion
@@ -498,7 +488,6 @@ function createMANAGERSEARCH() {
add_elastalert_to_minion
add_kibana_to_minion
add_redis_to_minion
add_curator_to_minion
add_telegraf_to_minion
add_influxdb_to_minion
add_nginx_to_minion
@@ -548,7 +537,6 @@ function createHEAVYNODE() {
add_sensor_to_minion
add_strelka_to_minion
add_redis_to_minion
add_curator_to_minion
add_telegraf_to_minion
}

5
salt/manager/tools/sbin/soup
View File

@@ -465,9 +465,8 @@ post_to_2.4.30() {
}
post_to_2.4.40() {
echo "Removing Curator close and delete files"
rm -f /opt/so/conf/curator/action/*-close.y*ml
rm -f /opt/so/conf/curator/action/logs*-delete.y*ml
echo "Removing Curator configuration"
rm -rf /opt/so/conf/curator/
}
repo_sync() {

5
salt/top.sls
View File

@@ -74,7 +74,6 @@ base:
- suricata
- zeek
- strelka
- curator
- elastalert
- utility
- soctopus
@@ -104,7 +103,6 @@ base:
- redis
- elastic-fleet-package-registry
- kibana
- curator
- elastalert
- utility
- soctopus
@@ -140,7 +138,6 @@ base:
- suricata
- zeek
- strelka
- curator
- elastalert
- utility
- soctopus
@@ -179,7 +176,6 @@ base:
- elasticsearch
- logstash
- redis
- curator
- elastic-fleet-package-registry
- kibana
- elastalert
@@ -199,7 +195,6 @@ base:
- elasticsearch
- logstash
- redis
- curator
- strelka
- pcap
- suricata

3
setup/so-functions
View File

@@ -1412,7 +1412,7 @@ make_some_dirs() {
mkdir -p $local_salt_dir/salt/firewall/portgroups
mkdir -p $local_salt_dir/salt/firewall/ports
for THEDIR in bpf pcap elasticsearch ntp firewall redis backup influxdb strelka sensoroni curator soc soctopus docker zeek suricata nginx telegraf logstash soc manager kratos idstools idh elastalert global;do
for THEDIR in bpf pcap elasticsearch ntp firewall redis backup influxdb strelka sensoroni soc soctopus docker zeek suricata nginx telegraf logstash soc manager kratos idstools idh elastalert global;do
mkdir -p $local_salt_dir/pillar/$THEDIR
touch $local_salt_dir/pillar/$THEDIR/adv_$THEDIR.sls
touch $local_salt_dir/pillar/$THEDIR/soc_$THEDIR.sls
@@ -1555,7 +1555,6 @@ reserve_group_ids() {
logCmd "groupadd -g 931 logstash"
logCmd "groupadd -g 932 kibana"
logCmd "groupadd -g 933 elastalert"
logCmd "groupadd -g 934 curator"
logCmd "groupadd -g 937 zeek"
logCmd "groupadd -g 940 suricata"
logCmd "groupadd -g 941 stenographer"

8
setup/so-variables
View File

@@ -1,4 +1,4 @@
#!/bin/bash
a!/bin/bash
SOVERSION=$(cat ../VERSION)
mkdir -p /nsm
@@ -112,12 +112,6 @@ export sensoroni_pillar_file
adv_sensoroni_pillar_file="$local_salt_dir/pillar/sensoroni/adv_sensoroni.sls"
export adv_sensoroni_pillar_file
curator_pillar_file="$local_salt_dir/pillar/curator/soc_curator.sls"
export curator_pillar_file
adv_curator_pillar_file="$local_salt_dir/pillar/curator/adv_curator.sls"
export adv_curator_pillar_file
soctopus_pillar_file="$local_salt_dir/pillar/soctopus/soc_soctopus.sls"
export soctopus_pillar_file