configure and enable/disable curator in ui

2025-12-06 09:12:45 +01:00 · 2023-05-11 10:17:28 -04:00
parent 06a0492226
commit c74b440922
11 changed files with 443 additions and 312 deletions
--- a/salt/allowed_states.map.jinja
+++ b/salt/allowed_states.map.jinja
@@ -3,7 +3,6 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.

-{% set CURATOR = salt['pillar.get']('curator:enabled', True) %}
 {% set ISAIRGAP = salt['pillar.get']('global:airgap', False) %}
 {% import_yaml 'salt/minion.defaults.yaml' as saltversion %}
 {% set saltversion = saltversion.salt.minion.version %}
--- a/salt/curator/config.sls
+++ b/salt/curator/config.sls
@@ -0,0 +1,81 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{% from 'allowed_states.map.jinja' import allowed_states %}
+{% if sls.split('.')[0] in allowed_states %}
+{%   from "curator/map.jinja" import CURATORMERGED %}
+
+# Create the group
+curatorgroup:
+  group.present:
+    - name: curator
+    - gid: 934
+
+# Add user
+curator:
+  user.present:
+    - uid: 934
+    - gid: 934
+    - home: /opt/so/conf/curator
+    - createhome: False
+
+# Create the log directory
+curlogdir:
+  file.directory:
+    - name: /opt/so/log/curator
+    - user: 934
+    - group: 939
+
+curactiondir:
+  file.directory:
+    - name: /opt/so/conf/curator/action
+    - user: 934
+    - group: 939
+    - makedirs: True
+
+actionconfs:
+  file.recurse:
+    - name: /opt/so/conf/curator/action
+    - source: salt://curator/files/action
+    - user: 934
+    - group: 939
+    - template: jinja
+    - defaults:
+        CURATORMERGED: {{ CURATORMERGED.elasticsearch.index_settings }}
+        
+curconf:
+  file.managed:
+    - name: /opt/so/conf/curator/curator.yml
+    - source: salt://curator/files/curator.yml
+    - user: 934
+    - group: 939
+    - mode: 660
+    - template: jinja
+    - show_changes: False
+
+curator_sbin:
+  file.recurse:
+    - name: /usr/sbin
+    - source: salt://curator/tools/sbin
+    - user: 934
+    - group: 939
+    - file_mode: 755
+
+curator_sbin_jinja:
+  file.recurse:
+    - name: /usr/sbin
+    - source: salt://curator/tools/sbin_jinja
+    - user: 934
+    - group: 939 
+    - file_mode: 755
+    - template: jinja
+
+{% else %}
+
+{{sls}}_state_not_allowed:
+  test.fail_without_changes:
+    - name: {{sls}}_state_not_allowed
+
+{% endif %}
--- a/salt/curator/defaults.yaml
+++ b/salt/curator/defaults.yaml
@@ -1,4 +1,6 @@
-elasticsearch:
+curator:
+  enabled: False
+  elasticsearch:
    index_settings:
      logs-import-so:
        close: 73000
--- a/salt/curator/disabled.sls
+++ b/salt/curator/disabled.sls
@@ -0,0 +1,35 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{% from 'allowed_states.map.jinja' import allowed_states %}
+{% if sls.split('.')[0] in allowed_states %}
+
+include:
+  - curator.sostatus
+  
+so-curator:
+  docker_container.absent:
+    - force: True
+
+so-curator_so-status.disabled:
+  file.comment:
+    - name: /opt/so/conf/so-status/so-status.conf
+    - regex: ^so-curator$
+
+so-curator-cluster-close:
+  cron.absent:
+    - identifier: so-curator-cluster-close
+
+so-curator-cluster-delete:
+  cron.absent:
+    - identifier: so-curator-cluster-delete
+
+{% else %}
+
+{{sls}}_state_not_allowed:
+  test.fail_without_changes:
+    - name: {{sls}}_state_not_allowed
+
+{% endif %}
--- a/salt/curator/enabled.sls
+++ b/salt/curator/enabled.sls
@@ -0,0 +1,71 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{% from 'allowed_states.map.jinja' import allowed_states %}
+{% if sls.split('.')[0] in allowed_states %}
+{%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   from 'docker/docker.map.jinja' import DOCKER %}
+
+include:
+  - curator.config
+  - curator.sostatus
+
+so-curator:
+  docker_container.running:
+    - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-curator:{{ GLOBALS.so_version }}
+    - start: True
+    - hostname: curator
+    - name: so-curator
+    - user: curator
+    - networks:
+      - sobridge:
+        - ipv4_address: {{ DOCKER.containers['so-curator'].ip }}
+    - interactive: True
+    - tty: True
+    - binds:
+      - /opt/so/conf/curator/curator.yml:/etc/curator/config/curator.yml:ro
+      - /opt/so/conf/curator/action/:/etc/curator/action:ro
+      - /opt/so/log/curator:/var/log/curator:rw
+    - require:
+      - file: actionconfs
+      - file: curconf
+      - file: curlogdir
+    - watch:
+      - file: curconf
+
+delete_so-curator_so-status.disabled:
+  file.uncomment:
+    - name: /opt/so/conf/so-status/so-status.conf
+    - regex: ^so-curator$
+
+so-curator-cluster-close:
+  cron.present:
+    - name: /usr/sbin/so-curator-cluster-close > /opt/so/log/curator/cron-close.log 2>&1
+    - identifier: so-curator-cluster-close
+    - user: root
+    - minute: '2'
+    - hour: '*/1'
+    - daymonth: '*'
+    - month: '*'
+    - dayweek: '*'
+
+so-curator-cluster-delete:
+  cron.present:
+    - name: /usr/sbin/so-curator-cluster-delete > /opt/so/log/curator/cron-cluster-delete.log 2>&1
+    - identifier: so-curator-cluster-delete
+    - user: root
+    - minute: '*/5'
+    - hour: '*'
+    - daymonth: '*'
+    - month: '*'
+    - dayweek: '*'
+
+{% else %}
+
+{{sls}}_state_not_allowed:
+  test.fail_without_changes:
+    - name: {{sls}}_state_not_allowed
+
+{% endif %}
--- a/salt/curator/files/curator.yml
+++ b/salt/curator/files/curator.yml
@@ -4,9 +4,9 @@
 # Elastic License 2.0.

 {% from 'vars/globals.map.jinja' import GLOBALS %}
-{% if grains['role'] in ['so-searchnode', 'so-heavynode'] %}
+{% if GLOBALS.role in ['so-searchnode', 'so-heavynode'] %}
  {%- set elasticsearch = GLOBALS.node_ip -%}
-{% elif grains['role'] in ['so-eval', 'so-managersearch', 'so-standalone', 'so-manager'] %}
+{% elif GLOBALS.role in ['so-eval', 'so-managersearch', 'so-standalone', 'so-manager'] %}
  {%- set elasticsearch = GLOBALS.manager_ip -%}
 {%- endif %}
 {%-   set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %}
@@ -30,10 +30,8 @@ elasticsearch:
      id:
      api_key:
    master_only: False
-{%- if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %}
    username: "{{ ES_USER }}"
    password: "{{ ES_PASS }}"
-{%- endif %}

 logging:
  loglevel: INFO
--- a/salt/curator/init.sls
+++ b/salt/curator/init.sls
@@ -3,146 +3,11 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.

-{% from 'allowed_states.map.jinja' import allowed_states %}
-{% if sls in allowed_states %}
-{% from 'vars/globals.map.jinja' import GLOBALS %}
-{% from 'docker/docker.map.jinja' import DOCKER %}
-{% from "curator/map.jinja" import CURATOROPTIONS %}
-{% from "curator/map.jinja" import CURATORMERGED %}
-{% set REMOVECURATORCRON = False %}
-
-# Curator
-# Create the group
-curatorgroup:
-  group.present:
-    - name: curator
-    - gid: 934
-
-# Add user
-curator:
-  user.present:
-    - uid: 934
-    - gid: 934
-    - home: /opt/so/conf/curator
-    - createhome: False
-
-# Create the log directory
-curlogdir:
-  file.directory:
-    - name: /opt/so/log/curator
-    - user: 934
-    - group: 939
-
-curactiondir:
-  file.directory:
-    - name: /opt/so/conf/curator/action
-    - user: 934
-    - group: 939
-    - makedirs: True
-
-actionconfs:
-  file.recurse:
-    - name: /opt/so/conf/curator/action
-    - source: salt://curator/files/action
-    - user: 934
-    - group: 939
-    - template: jinja
-    - defaults:
-        CURATORMERGED: {{ CURATORMERGED }}
-        
-curconf:
-  file.managed:
-    - name: /opt/so/conf/curator/curator.yml
-    - source: salt://curator/files/curator.yml
-    - user: 934
-    - group: 939
-    - mode: 660
-    - template: jinja
-    - show_changes: False
-
-curator_sbin:
-  file.recurse:
-    - name: /usr/sbin
-    - source: salt://curator/tools/sbin
-    - user: 934
-    - group: 939
-    - file_mode: 755
-
-curator_sbin_jinja:
-  file.recurse:
-    - name: /usr/sbin
-    - source: salt://curator/tools/sbin_jinja
-    - user: 934
-    - group: 939 
-    - file_mode: 755
-    - template: jinja
-
-so-curator:
-  docker_container.{{ CURATOROPTIONS.status }}:
-    - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-curator:{{ GLOBALS.so_version }}
-    - start: {{ CURATOROPTIONS.start }}
-    - hostname: curator
-    - name: so-curator
-    - user: curator
-    - networks:
-      - sobridge:
-        - ipv4_address: {{ DOCKER.containers['so-curator'].ip }}
-    - interactive: True
-    - tty: True
-    - binds:
-      - /opt/so/conf/curator/curator.yml:/etc/curator/config/curator.yml:ro
-      - /opt/so/conf/curator/action/:/etc/curator/action:ro
-      - /opt/so/log/curator:/var/log/curator:rw
-    - require:
-      - file: actionconfs
-      - file: curconf
-      - file: curlogdir
-    - watch:
-      - file: curconf
-
-append_so-curator_so-status.conf:
-  file.append:
-    - name: /opt/so/conf/so-status/so-status.conf
-    - text: so-curator
-    - unless: grep -q so-curator /opt/so/conf/so-status/so-status.conf
-  {% if not CURATOROPTIONS.start %}
-so-curator_so-status.disabled:
-  file.comment:
-    - name: /opt/so/conf/so-status/so-status.conf
-    - regex: ^so-curator$
-  {% else %}
-delete_so-curator_so-status.disabled:
-  file.uncomment:
-    - name: /opt/so/conf/so-status/so-status.conf
-    - regex: ^so-curator$
-  {% endif %}
-
-so-curator-cluster-close:
-  cron.present:
-    - name: /usr/sbin/so-curator-cluster-close > /opt/so/log/curator/cron-close.log 2>&1
-    - identifier: so-curator-cluster-close
-    - user: root
-    - minute: '2'
-    - hour: '*/1'
-    - daymonth: '*'
-    - month: '*'
-    - dayweek: '*'
-
-so-curator-cluster-delete:
-  cron.present:
-    - name: /usr/sbin/so-curator-cluster-delete > /opt/so/log/curator/cron-cluster-delete.log 2>&1
-    - identifier: so-curator-cluster-delete
-    - user: root
-    - minute: '*/5'
-    - hour: '*'
-    - daymonth: '*'
-    - month: '*'
-    - dayweek: '*'
+{% from 'curator/map.jinja' import CURATORMERGED %}

+include:
+{% if CURATORMERGED.enabled %}
+  - curator.enabled
 {% else %}
-
-{{sls}}_state_not_allowed:
-  test.fail_without_changes:
-    - name: {{sls}}_state_not_allowed
-
+  - curator.disabled
 {% endif %}
--- a/salt/curator/map.jinja
+++ b/salt/curator/map.jinja
@@ -1,18 +1,7 @@
-{% set CURATOROPTIONS = {} %}
-{% set ENABLED = salt['pillar.get']('curator:enabled', True) %}
-{% do CURATOROPTIONS.update({'manage_sostatus': True}) %}
-
-# don't start the docker container if curator is disabled via pillar
-{% if not ENABLED %}
-  {% do CURATOROPTIONS.update({'start': False}) %}
-  {% do CURATOROPTIONS.update({'status': 'absent'}) %}
-  {% if (TRUECLUSTER and grains.id.split('_')|last == 'searchnode') or (not TRUECLUSTER and grains.id.split('_')|last == 'manager') %}
-    {% do CURATOROPTIONS.update({'manage_sostatus': False}) %}
-  {% endif %}
-{% else %}
-  {% do CURATOROPTIONS.update({'start': True}) %}
-  {% do CURATOROPTIONS.update({'status': 'running'}) %}
-{% endif %}
+{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+   or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+   https://securityonion.net/license; you may not use this file except in compliance with the
+   Elastic License 2.0. #}

 {% import_yaml 'curator/defaults.yaml' as CURATORDEFAULTS %}
-{% set CURATORMERGED = salt['pillar.get']('elasticsearch:index_settings', CURATORDEFAULTS.elasticsearch.index_settings, merge=true) %}
+{% set CURATORMERGED = salt['pillar.get']('curator', CURATORDEFAULTS.curator, merge=true) %}
--- a/salt/curator/soc_curator.yaml
+++ b/salt/curator/soc_curator.yaml
@@ -0,0 +1,108 @@
+curator:
+  enabled:
+    description: You can enable or disable Curator.
+    helpLink: curator.html
+  elasticsearch:
+    index_settings:
+      logs-import-so:
+        close: &close
+          description: Age, in days, when Curator closes the index.
+          helpLink: curator.html
+          forcedType: int
+        delete: &delete
+          description: Age, in days, when Curator deletes the index.
+          helpLink: curator.html
+          forcedType: int
+      logs-strelka-so:
+        close: *close
+        delete: *delete
+      logs-suricata-so:
+        close: *close
+        delete: *delete
+      logs-syslog-so:
+        close: *close
+        delete: *delete
+      logs-zeek-so:
+        close: *close
+        delete: *delete
+      logs-elastic_agent-metricbeat-default:
+        close: *close
+        delete: *delete
+      logs-elastic_agent-osquerybeat-default:
+        close: *close
+        delete: *delete
+      logs-elastic_agent-fleet_server-default:
+        close: *close
+        delete: *delete
+      logs-elastic_agent-filebeat-default:
+        close: *close
+        delete: *delete
+      logs-elastic_agent-default:
+        close: *close
+        delete: *delete
+      logs-system-auth-default:
+        close: *close
+        delete: *delete
+      logs-system-application-default:
+        close: *close
+        delete: *delete
+      logs-system-security-default:
+        close: *close
+        delete: *delete
+      logs-system-system-default:
+        close: *close
+        delete: *delete
+      logs-system-syslog-default:
+        close: *close
+        delete: *delete
+      logs-windows-powershell-default:
+        close: *close
+        delete: *delete
+      logs-windows-sysmon_operational-default:
+        close: *close
+        delete: *delete
+      so-beats:
+        close: *close
+        delete: *delete
+      so-elasticsearch:
+        close: *close
+        delete: *delete
+      so-firewall:
+        close: *close
+        delete: *delete
+      so-ids:
+        close: *close
+        delete: *delete
+      so-import:
+        close: *close
+        delete: *delete
+      so-kratos:
+        close: *close
+        delete: *delete
+      so-kibana:
+        close: *close
+        delete: *delete
+      so-logstash:
+        close: *close
+        delete: *delete
+      so-netflow:
+        close: *close
+        delete: *delete
+      so-osquery:
+        close: *close
+        delete: *delete
+      so-ossec:
+        close: *close
+        delete: *delete
+      so-redis:
+        close: *close
+        delete: *delete
+      so-strelka:
+        close: *close
+        delete: *delete
+      so-syslog:
+        close: *close
+        delete: *delete
+      so-zeek:
+        close: *close
+        delete: *delete
--- a/salt/curator/sostatus.sls
+++ b/salt/curator/sostatus.sls
@@ -0,0 +1,21 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{% from 'allowed_states.map.jinja' import allowed_states %}
+{% if sls.split('.')[0] in allowed_states %}
+
+append_so-curator_so-status.conf:
+  file.append:
+    - name: /opt/so/conf/so-status/so-status.conf
+    - text: so-curator
+    - unless: grep -q so-curator /opt/so/conf/so-status/so-status.conf
+
+{% else %}
+
+{{sls}}_state_not_allowed:
+  test.fail_without_changes:
+    - name: {{sls}}_state_not_allowed
+
+{% endif %}
--- a/salt/manager/tools/sbin/so-minion
+++ b/salt/manager/tools/sbin/so-minion
@@ -249,50 +249,27 @@ function add_redis_to_minion() {
 	"  " >> $PILLARFILE
 }

-function add_strelka_backend_to_minion() {
+function add_strelka_strelka_to_minion() {
    printf '%s\n'\
    "strelka:"\
 	"  backend:"\
    "    enabled: True"\
-	"  " >> $PILLARFILE
-}
-
-function add_strelka_filestream_to_minion() {
-    printf '%s\n'\
-    "strelka:"\
 	"  filestream:"\
    "    enabled: True"\
-	"  " >> $PILLARFILE
-}
-
-function add_strelka_frontend_to_minion() {
-    printf '%s\n'\
-    "strelka:"\
 	"  frontend:"\
    "    enabled: True"\
-	"  " >> $PILLARFILE
-}
-
-function add_strelka_manager_to_minion() {
-    printf '%s\n'\
-    "strelka:"\
 	"  manager:"\
    "    enabled: True"\
-	"  " >> $PILLARFILE
-}
-
-function add_strelka_coordinator_to_minion() {
-    printf '%s\n'\
-    "strelka:"\
 	"  coordinator:"\
+    "    enabled: True"\
+	"  gatekeeper:"\
    "    enabled: True"\
 	"  " >> $PILLARFILE
 }

-function add_strelka_gatekeeper_to_minion() {
+function add_curator_to_minion() {
    printf '%s\n'\
-    "strelka:"\
-	"  gatekeeper:"\
+    "curator:"\
    "  enabled: True"\
 	"  " >> $PILLARFILE
 }
@@ -343,30 +320,22 @@ function apply_ES_state() {
 function createEVAL() {
 	add_elasticsearch_to_minion
 	add_sensor_to_minion
-	add_strelka_backend_to_minion
-    add_strelka_filestream_to_minion
-    add_strelka_frontend_to_minion
-    add_strelka_manager_to_minion
-    add_strelka_coordinator_to_minion
-    add_strelka_gatekeeper_to_minion
+	add_strelka_strelka_to_minion
 	add_elastalert_to_minion
 	add_kibana_to_minion
+	add_curator_to_minion
 }

 function createSTANDALONE() {
 	add_elasticsearch_to_minion
 	add_logstash_to_minion
 	add_sensor_to_minion
-	add_strelka_backend_to_minion
-    add_strelka_filestream_to_minion
-    add_strelka_frontend_to_minion
-    add_strelka_manager_to_minion
-    add_strelka_coordinator_to_minion
-    add_strelka_gatekeeper_to_minion
+	add_strelka_strelka_to_minion
 	add_playbook_to_minion
 	add_elastalert_to_minion
 	add_kibana_to_minion
 	add_redis_to_minion
+	add_curator_to_minion
 }

 function createMANAGER() {
@@ -376,6 +345,7 @@ function createMANAGER() {
 	add_elastalert_to_minion
 	add_kibana_to_minion
 	add_redis_to_minion
+	add_curator_to_minion
 }

 function createMANAGERSEARCH() {
@@ -385,6 +355,7 @@ function createMANAGERSEARCH() {
 	add_elastalert_to_minion
 	add_kibana_to_minion
 	add_redis_to_minion
+	add_curator_to_minion
 }

 function createIMPORT() {
@@ -409,23 +380,14 @@ function createHEAVYNODE() {
 	add_elasticsearch_to_minion
 	add_logstash_to_minion
 	add_sensor_to_minion
-	add_strelka_backend_to_minion
-    add_strelka_filestream_to_minion
-    add_strelka_frontend_to_minion
-    add_strelka_manager_to_minion
-    add_strelka_coordinator_to_minion
-    add_strelka_gatekeeper_to_minion
+	add_strelka_strelka_to_minion
 	add_redis_to_minion
+	add_curator_to_minion
 }

 function createSENSOR() {
 	add_sensor_to_minion
-	add_strelka_backend_to_minion
-    add_strelka_filestream_to_minion
-    add_strelka_frontend_to_minion
-    add_strelka_manager_to_minion
-    add_strelka_coordinator_to_minion
-    add_strelka_gatekeeper_to_minion
+	add_strelka_strelka_to_minion
 }

 function createSEARCHNODE() {