From 6d7243038ce792349d4375d7ad05497c91ef6dc4 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 27 Oct 2023 11:20:49 -0400 Subject: [PATCH 01/21] switch back to kilo version --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 8ea99f559..7f2e97617 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4.30 +2.4.0-kilo From d203aec44a47f61e7d75fa19250fad01ebd12821 Mon Sep 17 00:00:00 2001 From: Wes Date: Fri, 8 Dec 2023 19:37:06 +0000 Subject: [PATCH 02/21] Remove Curator --- pillar/top.sls | 10 --- salt/allowed_states.map.jinja | 4 - salt/common/tools/sbin/so-image-common | 1 - salt/curator/config.sls | 81 ------------------- salt/curator/defaults.yaml | 34 -------- salt/curator/disabled.sls | 35 -------- salt/curator/enabled.sls | 81 ------------------- salt/curator/files/action/delete.yml | 32 -------- salt/curator/files/action/so-beats-delete.yml | 27 ------- .../files/action/so-elasticsearch-delete.yml | 27 ------- .../files/action/so-firewall-delete.yml | 28 ------- salt/curator/files/action/so-ids-delete.yml | 28 ------- .../curator/files/action/so-import-delete.yml | 27 ------- .../curator/files/action/so-kibana-delete.yml | 27 ------- .../curator/files/action/so-kratos-delete.yml | 27 ------- .../files/action/so-logstash-delete.yml | 27 ------- .../files/action/so-netflow-delete.yml | 27 ------- .../files/action/so-osquery-delete.yml | 27 ------- salt/curator/files/action/so-ossec-delete.yml | 27 ------- salt/curator/files/action/so-redis-delete.yml | 27 ------- .../files/action/so-strelka-delete.yml | 27 ------- .../curator/files/action/so-syslog-delete.yml | 27 ------- salt/curator/files/action/so-zeek-delete.yml | 27 ------- salt/curator/files/curator.yml | 40 --------- salt/curator/init.sls | 13 --- salt/curator/map.jinja | 7 -- salt/curator/soc_curator.yaml | 39 --------- salt/curator/sostatus.sls | 21 ----- salt/curator/tools/sbin/so-curator-delete | 17 ---- salt/curator/tools/sbin/so-curator-restart | 12 --- salt/curator/tools/sbin/so-curator-start | 12 --- salt/curator/tools/sbin/so-curator-stop | 12 --- salt/docker/defaults.yaml | 5 -- salt/docker/soc_docker.yaml | 1 - salt/elasticsearch/disabled.sls | 4 + salt/elasticsearch/enabled.sls | 20 +++++ .../sbin/so-elasticsearch-indices-delete} | 2 +- .../so-elasticsearch-indices-delete-delete} | 4 +- salt/firewall/containers.map.jinja | 3 - salt/logrotate/defaults.yaml | 2 +- salt/logrotate/soc_logrotate.yaml | 14 ++-- .../tools/sbin/so-elastic-auth-password-reset | 2 - salt/manager/tools/sbin/so-minion | 14 +--- salt/manager/tools/sbin/soup | 5 +- salt/top.sls | 5 -- setup/so-functions | 3 +- setup/so-variables | 8 +- 47 files changed, 40 insertions(+), 910 deletions(-) delete mode 100644 salt/curator/config.sls delete mode 100644 salt/curator/defaults.yaml delete mode 100644 salt/curator/disabled.sls delete mode 100644 salt/curator/enabled.sls delete mode 100644 salt/curator/files/action/delete.yml delete mode 100644 salt/curator/files/action/so-beats-delete.yml delete mode 100644 salt/curator/files/action/so-elasticsearch-delete.yml delete mode 100644 salt/curator/files/action/so-firewall-delete.yml delete mode 100644 salt/curator/files/action/so-ids-delete.yml delete mode 100644 salt/curator/files/action/so-import-delete.yml delete mode 100644 salt/curator/files/action/so-kibana-delete.yml delete mode 100644 salt/curator/files/action/so-kratos-delete.yml delete mode 100644 salt/curator/files/action/so-logstash-delete.yml delete mode 100644 salt/curator/files/action/so-netflow-delete.yml delete mode 100644 salt/curator/files/action/so-osquery-delete.yml delete mode 100644 salt/curator/files/action/so-ossec-delete.yml delete mode 100644 salt/curator/files/action/so-redis-delete.yml delete mode 100644 salt/curator/files/action/so-strelka-delete.yml delete mode 100644 salt/curator/files/action/so-syslog-delete.yml delete mode 100644 salt/curator/files/action/so-zeek-delete.yml delete mode 100644 salt/curator/files/curator.yml delete mode 100644 salt/curator/init.sls delete mode 100644 salt/curator/map.jinja delete mode 100644 salt/curator/soc_curator.yaml delete mode 100644 salt/curator/sostatus.sls delete mode 100644 salt/curator/tools/sbin/so-curator-delete delete mode 100644 salt/curator/tools/sbin/so-curator-restart delete mode 100644 salt/curator/tools/sbin/so-curator-start delete mode 100644 salt/curator/tools/sbin/so-curator-stop rename salt/{curator/tools/sbin/so-curator-cluster-delete => elasticsearch/tools/sbin/so-elasticsearch-indices-delete} (92%) rename salt/{curator/tools/sbin_jinja/so-curator-cluster-delete-delete => elasticsearch/tools/sbin_jinja/so-elasticsearch-indices-delete-delete} (96%) diff --git a/pillar/top.sls b/pillar/top.sls index 53ec8a330..ed3e49254 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -61,8 +61,6 @@ base: - elastalert.adv_elastalert - backup.soc_backup - backup.adv_backup - - curator.soc_curator - - curator.adv_curator - soctopus.soc_soctopus - soctopus.adv_soctopus - minions.{{ grains.id }} @@ -113,8 +111,6 @@ base: - kibana.adv_kibana - strelka.soc_strelka - strelka.adv_strelka - - curator.soc_curator - - curator.adv_curator - kratos.soc_kratos - kratos.adv_kratos - redis.soc_redis @@ -172,8 +168,6 @@ base: - kibana.adv_kibana - strelka.soc_strelka - strelka.adv_strelka - - curator.soc_curator - - curator.adv_curator - backup.soc_backup - backup.adv_backup - zeek.soc_zeek @@ -194,8 +188,6 @@ base: - logstash.adv_logstash - elasticsearch.soc_elasticsearch - elasticsearch.adv_elasticsearch - - curator.soc_curator - - curator.adv_curator - redis.soc_redis - redis.adv_redis - zeek.soc_zeek @@ -268,8 +260,6 @@ base: - soctopus.adv_soctopus - kibana.soc_kibana - kibana.adv_kibana - - curator.soc_curator - - curator.adv_curator - backup.soc_backup - backup.adv_backup - kratos.soc_kratos diff --git a/salt/allowed_states.map.jinja b/salt/allowed_states.map.jinja index 4e3e57f9c..6585bd96c 100644 --- a/salt/allowed_states.map.jinja +++ b/salt/allowed_states.map.jinja @@ -219,10 +219,6 @@ {% do allowed_states.append('kibana.secrets') %} {% endif %} - {% if grains.role in ['so-eval', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-manager'] %} - {% do allowed_states.append('curator') %} - {% endif %} - {% if grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-managersearch'] %} {% do allowed_states.append('elastalert') %} {% endif %} diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common index 7e510e3ad..7900b3c52 100755 --- a/salt/common/tools/sbin/so-image-common +++ b/salt/common/tools/sbin/so-image-common @@ -42,7 +42,6 @@ container_list() { ) elif [ $MANAGERCHECK != 'so-helix' ]; then TRUSTED_CONTAINERS=( - "so-curator" "so-elastalert" "so-elastic-agent" "so-elastic-agent-builder" diff --git a/salt/curator/config.sls b/salt/curator/config.sls deleted file mode 100644 index 89ff53b2a..000000000 --- a/salt/curator/config.sls +++ /dev/null @@ -1,81 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{% from 'allowed_states.map.jinja' import allowed_states %} -{% if sls.split('.')[0] in allowed_states %} -{% from "curator/map.jinja" import CURATORMERGED %} - -# Create the group -curatorgroup: - group.present: - - name: curator - - gid: 934 - -# Add user -curator: - user.present: - - uid: 934 - - gid: 934 - - home: /opt/so/conf/curator - - createhome: False - -# Create the log directory -curlogdir: - file.directory: - - name: /opt/so/log/curator - - user: 934 - - group: 939 - -curactiondir: - file.directory: - - name: /opt/so/conf/curator/action - - user: 934 - - group: 939 - - makedirs: True - -actionconfs: - file.recurse: - - name: /opt/so/conf/curator/action - - source: salt://curator/files/action - - user: 934 - - group: 939 - - template: jinja - - defaults: - CURATORMERGED: {{ CURATORMERGED.elasticsearch.index_settings }} - -curconf: - file.managed: - - name: /opt/so/conf/curator/curator.yml - - source: salt://curator/files/curator.yml - - user: 934 - - group: 939 - - mode: 660 - - template: jinja - - show_changes: False - -curator_sbin: - file.recurse: - - name: /usr/sbin - - source: salt://curator/tools/sbin - - user: 934 - - group: 939 - - file_mode: 755 - -curator_sbin_jinja: - file.recurse: - - name: /usr/sbin - - source: salt://curator/tools/sbin_jinja - - user: 934 - - group: 939 - - file_mode: 755 - - template: jinja - -{% else %} - -{{sls}}_state_not_allowed: - test.fail_without_changes: - - name: {{sls}}_state_not_allowed - -{% endif %} diff --git a/salt/curator/defaults.yaml b/salt/curator/defaults.yaml deleted file mode 100644 index b2be8d5ed..000000000 --- a/salt/curator/defaults.yaml +++ /dev/null @@ -1,34 +0,0 @@ -curator: - enabled: False - elasticsearch: - index_settings: - so-beats: - delete: 365 - so-elasticsearch: - delete: 365 - so-firewall: - delete: 365 - so-ids: - delete: 365 - so-import: - delete: 73001 - so-kratos: - delete: 365 - so-kibana: - delete: 365 - so-logstash: - delete: 365 - so-netflow: - delete: 365 - so-osquery: - delete: 365 - so-ossec: - delete: 365 - so-redis: - delete: 365 - so-strelka: - delete: 365 - so-syslog: - delete: 365 - so-zeek: - delete: 365 diff --git a/salt/curator/disabled.sls b/salt/curator/disabled.sls deleted file mode 100644 index acf9e3701..000000000 --- a/salt/curator/disabled.sls +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{% from 'allowed_states.map.jinja' import allowed_states %} -{% if sls.split('.')[0] in allowed_states %} - -include: - - curator.sostatus - -so-curator: - docker_container.absent: - - force: True - -so-curator_so-status.disabled: - file.comment: - - name: /opt/so/conf/so-status/so-status.conf - - regex: ^so-curator$ - -so-curator-cluster-close: - cron.absent: - - identifier: so-curator-cluster-close - -so-curator-cluster-delete: - cron.absent: - - identifier: so-curator-cluster-delete - -{% else %} - -{{sls}}_state_not_allowed: - test.fail_without_changes: - - name: {{sls}}_state_not_allowed - -{% endif %} diff --git a/salt/curator/enabled.sls b/salt/curator/enabled.sls deleted file mode 100644 index 916aa920d..000000000 --- a/salt/curator/enabled.sls +++ /dev/null @@ -1,81 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{% from 'allowed_states.map.jinja' import allowed_states %} -{% if sls.split('.')[0] in allowed_states %} -{% from 'vars/globals.map.jinja' import GLOBALS %} -{% from 'docker/docker.map.jinja' import DOCKER %} - -include: - - curator.config - - curator.sostatus - -so-curator: - docker_container.running: - - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-curator:{{ GLOBALS.so_version }} - - start: True - - hostname: curator - - name: so-curator - - user: curator - - networks: - - sobridge: - - ipv4_address: {{ DOCKER.containers['so-curator'].ip }} - - interactive: True - - tty: True - - binds: - - /opt/so/conf/curator/curator.yml:/etc/curator/config/curator.yml:ro - - /opt/so/conf/curator/action/:/etc/curator/action:ro - - /opt/so/log/curator:/var/log/curator:rw - {% if DOCKER.containers['so-curator'].custom_bind_mounts %} - {% for BIND in DOCKER.containers['so-curator'].custom_bind_mounts %} - - {{ BIND }} - {% endfor %} - {% endif %} - {% if DOCKER.containers['so-curator'].extra_hosts %} - - extra_hosts: - {% for XTRAHOST in DOCKER.containers['so-curator'].extra_hosts %} - - {{ XTRAHOST }} - {% endfor %} - {% endif %} - {% if DOCKER.containers['so-curator'].extra_env %} - - environment: - {% for XTRAENV in DOCKER.containers['so-curator'].extra_env %} - - {{ XTRAENV }} - {% endfor %} - {% endif %} - - require: - - file: actionconfs - - file: curconf - - file: curlogdir - - watch: - - file: curconf - -delete_so-curator_so-status.disabled: - file.uncomment: - - name: /opt/so/conf/so-status/so-status.conf - - regex: ^so-curator$ - -so-curator-cluster-close: - cron.absent: - - identifier: so-curator-cluster-close - -so-curator-cluster-delete: - cron.present: - - name: /usr/sbin/so-curator-cluster-delete > /opt/so/log/curator/cron-cluster-delete.log 2>&1 - - identifier: so-curator-cluster-delete - - user: root - - minute: '*/5' - - hour: '*' - - daymonth: '*' - - month: '*' - - dayweek: '*' - -{% else %} - -{{sls}}_state_not_allowed: - test.fail_without_changes: - - name: {{sls}}_state_not_allowed - -{% endif %} diff --git a/salt/curator/files/action/delete.yml b/salt/curator/files/action/delete.yml deleted file mode 100644 index 253c6fd67..000000000 --- a/salt/curator/files/action/delete.yml +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{% import_yaml 'elasticsearch/defaults.yaml' as ELASTICDEFAULTS %} -{% set ELASTICMERGED = salt['pillar.get']('elasticsearch:retention', ELASTICDEFAULTS.elasticsearch.retention, merge=true) %} - -{{ ELASTICMERGED.retention_pct }} - -{%- set log_size_limit = salt['pillar.get']('elasticsearch:log_size_limit') %} -actions: - 1: - action: delete_indices - description: >- - Delete indices when {{log_size_limit}}(GB) is exceeded. - options: - allow_ilm_indices: True - ignore_empty_list: True - disable_action: False - filters: - - filtertype: pattern - kind: regex - value: '^(logstash-.*|so-.*|.ds-logs-.*-so.*)$' - - filtertype: pattern - kind: regex - value: '^(so-case.*)$' - exclude: True - - filtertype: space - source: creation_date - use_age: True - disk_space: {{log_size_limit}} diff --git a/salt/curator/files/action/so-beats-delete.yml b/salt/curator/files/action/so-beats-delete.yml deleted file mode 100644 index c4e1f8b4e..000000000 --- a/salt/curator/files/action/so-beats-delete.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{%- set DELETE_DAYS = CURATORMERGED['so-beats'].delete %} -actions: - 1: - action: delete_indices - description: >- - Delete beats indices when older than {{ DELETE_DAYS }} days. - options: - ignore_empty_list: True - disable_action: False - filters: - - filtertype: pattern - kind: regex - value: '^(logstash-beats.*|so-beats.*)$' - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: {{ DELETE_DAYS }} - exclude: - - \ No newline at end of file diff --git a/salt/curator/files/action/so-elasticsearch-delete.yml b/salt/curator/files/action/so-elasticsearch-delete.yml deleted file mode 100644 index 3c6bf4aac..000000000 --- a/salt/curator/files/action/so-elasticsearch-delete.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{%- set DELETE_DAYS = CURATORMERGED['so-elasticsearch'].delete %} -actions: - 1: - action: delete_indices - description: >- - Delete elasticsearch indices when older than {{ DELETE_DAYS }} days. - options: - ignore_empty_list: True - disable_action: False - filters: - - filtertype: pattern - kind: regex - value: '^(logstash-elasticsearch.*|so-elasticsearch.*)$' - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: {{ DELETE_DAYS }} - exclude: - - \ No newline at end of file diff --git a/salt/curator/files/action/so-firewall-delete.yml b/salt/curator/files/action/so-firewall-delete.yml deleted file mode 100644 index 5143e2fe9..000000000 --- a/salt/curator/files/action/so-firewall-delete.yml +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - - -{%- set DELETE_DAYS = CURATORMERGED['so-firewall'].delete %} -actions: - 1: - action: delete_indices - description: >- - Delete firewall indices when older than {{ DELETE_DAYS }} days. - options: - ignore_empty_list: True - disable_action: False - filters: - - filtertype: pattern - kind: regex - value: '^(logstash-firewall.*|so-firewall.*)$' - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: {{ DELETE_DAYS }} - exclude: - - \ No newline at end of file diff --git a/salt/curator/files/action/so-ids-delete.yml b/salt/curator/files/action/so-ids-delete.yml deleted file mode 100644 index 6cf120fef..000000000 --- a/salt/curator/files/action/so-ids-delete.yml +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - - -{%- set DELETE_DAYS = CURATORMERGED['so-ids'].delete %} -actions: - 1: - action: delete_indices - description: >- - Delete IDS indices when older than {{ DELETE_DAYS }} days. - options: - ignore_empty_list: True - disable_action: False - filters: - - filtertype: pattern - kind: regex - value: '^(logstash-ids.*|so-ids.*)$' - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: {{ DELETE_DAYS }} - exclude: - - \ No newline at end of file diff --git a/salt/curator/files/action/so-import-delete.yml b/salt/curator/files/action/so-import-delete.yml deleted file mode 100644 index 36e213b26..000000000 --- a/salt/curator/files/action/so-import-delete.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{%- set DELETE_DAYS = CURATORMERGED['so-import'].delete %} -actions: - 1: - action: delete_indices - description: >- - Delete import indices when older than {{ DELETE_DAYS }} days. - options: - ignore_empty_list: True - disable_action: False - filters: - - filtertype: pattern - kind: regex - value: '^(logstash-import.*|so-import.*)$' - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: {{ DELETE_DAYS }} - exclude: - - \ No newline at end of file diff --git a/salt/curator/files/action/so-kibana-delete.yml b/salt/curator/files/action/so-kibana-delete.yml deleted file mode 100644 index 971a178fe..000000000 --- a/salt/curator/files/action/so-kibana-delete.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{%- set DELETE_DAYS = CURATORMERGED['so-kibana'].delete %} -actions: - 1: - action: delete_indices - description: >- - Delete kibana indices when older than {{ DELETE_DAYS }} days. - options: - ignore_empty_list: True - disable_action: False - filters: - - filtertype: pattern - kind: regex - value: '^(logstash-kibana.*|so-kibana.*)$' - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: {{ DELETE_DAYS }} - exclude: - - \ No newline at end of file diff --git a/salt/curator/files/action/so-kratos-delete.yml b/salt/curator/files/action/so-kratos-delete.yml deleted file mode 100644 index d7cb2c4ad..000000000 --- a/salt/curator/files/action/so-kratos-delete.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{%- set DELETE_DAYS = CURATORMERGED['so-kratos'].delete %} -actions: - 1: - action: delete_indices - description: >- - Delete kratos indices when older than {{ DELETE_DAYS }} days. - options: - ignore_empty_list: True - disable_action: False - filters: - - filtertype: pattern - kind: regex - value: '^(logstash-kratos.*|so-kratos.*)$' - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: {{ DELETE_DAYS }} - exclude: - - \ No newline at end of file diff --git a/salt/curator/files/action/so-logstash-delete.yml b/salt/curator/files/action/so-logstash-delete.yml deleted file mode 100644 index 1ca1a6f6c..000000000 --- a/salt/curator/files/action/so-logstash-delete.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{%- set DELETE_DAYS = CURATORMERGED['so-logstash'].delete %} -actions: - 1: - action: delete_indices - description: >- - Delete logstash indices when older than {{ DELETE_DAYS }} days. - options: - ignore_empty_list: True - disable_action: False - filters: - - filtertype: pattern - kind: regex - value: '^(logstash-logstash.*|so-logstash.*)$' - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: {{ DELETE_DAYS }} - exclude: - - \ No newline at end of file diff --git a/salt/curator/files/action/so-netflow-delete.yml b/salt/curator/files/action/so-netflow-delete.yml deleted file mode 100644 index 63adaa393..000000000 --- a/salt/curator/files/action/so-netflow-delete.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{%- set DELETE_DAYS = CURATORMERGED['so-netflow'].delete %} -actions: - 1: - action: delete_indices - description: >- - Delete netflow indices when older than {{ DELETE_DAYS }} days. - options: - ignore_empty_list: True - disable_action: False - filters: - - filtertype: pattern - kind: regex - value: '^(logstash-netflow.*|so-netflow.*)$' - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: {{ DELETE_DAYS }} - exclude: - - \ No newline at end of file diff --git a/salt/curator/files/action/so-osquery-delete.yml b/salt/curator/files/action/so-osquery-delete.yml deleted file mode 100644 index b6263b0e8..000000000 --- a/salt/curator/files/action/so-osquery-delete.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{%- set DELETE_DAYS = CURATORMERGED['so-osquery'].delete %} -actions: - 1: - action: delete_indices - description: >- - Delete import indices when older than {{ DELETE_DAYS }} days. - options: - ignore_empty_list: True - disable_action: False - filters: - - filtertype: pattern - kind: regex - value: '^(logstash-osquery.*|so-osquery.*)$' - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: {{ DELETE_DAYS }} - exclude: - - diff --git a/salt/curator/files/action/so-ossec-delete.yml b/salt/curator/files/action/so-ossec-delete.yml deleted file mode 100644 index e24fe3819..000000000 --- a/salt/curator/files/action/so-ossec-delete.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{%- set DELETE_DAYS = CURATORMERGED['so-ossec'].delete %} -actions: - 1: - action: delete_indices - description: >- - Delete ossec indices when older than {{ DELETE_DAYS }} days. - options: - ignore_empty_list: True - disable_action: False - filters: - - filtertype: pattern - kind: regex - value: '^(logstash-ossec.*|so-ossec.*)$' - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: {{ DELETE_DAYS }} - exclude: - - \ No newline at end of file diff --git a/salt/curator/files/action/so-redis-delete.yml b/salt/curator/files/action/so-redis-delete.yml deleted file mode 100644 index 1c7f95ded..000000000 --- a/salt/curator/files/action/so-redis-delete.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{%- set DELETE_DAYS = CURATORMERGED['so-redis'].delete %} -actions: - 1: - action: delete_indices - description: >- - Delete redis indices when older than {{ DELETE_DAYS }} days. - options: - ignore_empty_list: True - disable_action: False - filters: - - filtertype: pattern - kind: regex - value: '^(logstash-redis.*|so-redis.*)$' - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: {{ DELETE_DAYS }} - exclude: - - \ No newline at end of file diff --git a/salt/curator/files/action/so-strelka-delete.yml b/salt/curator/files/action/so-strelka-delete.yml deleted file mode 100644 index 90cf88e46..000000000 --- a/salt/curator/files/action/so-strelka-delete.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{%- set DELETE_DAYS = CURATORMERGED['so-strelka'].delete %} -actions: - 1: - action: delete_indices - description: >- - Delete Strelka indices when older than {{ DELETE_DAYS }} days. - options: - ignore_empty_list: True - disable_action: False - filters: - - filtertype: pattern - kind: regex - value: '^(logstash-strelka.*|so-strelka.*)$' - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: {{ DELETE_DAYS }} - exclude: - - \ No newline at end of file diff --git a/salt/curator/files/action/so-syslog-delete.yml b/salt/curator/files/action/so-syslog-delete.yml deleted file mode 100644 index c11d2ef5a..000000000 --- a/salt/curator/files/action/so-syslog-delete.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{%- set DELETE_DAYS = CURATORMERGED['so-syslog'].delete %} -actions: - 1: - action: delete_indices - description: >- - Delete syslog indices when older than {{ DELETE_DAYS }} days. - options: - ignore_empty_list: True - disable_action: False - filters: - - filtertype: pattern - kind: regex - value: '^(logstash-syslog.*|so-syslog.*)$' - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: {{ DELETE_DAYS }} - exclude: - - \ No newline at end of file diff --git a/salt/curator/files/action/so-zeek-delete.yml b/salt/curator/files/action/so-zeek-delete.yml deleted file mode 100644 index 1f8522696..000000000 --- a/salt/curator/files/action/so-zeek-delete.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{%- set DELETE_DAYS = CURATORMERGED['so-zeek'].delete %} -actions: - 1: - action: delete_indices - description: >- - Delete Zeek indices when older than {{ DELETE_DAYS }} days. - options: - ignore_empty_list: True - disable_action: False - filters: - - filtertype: pattern - kind: regex - value: '^(logstash-zeek.*|so-zeek.*)$' - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: {{ DELETE_DAYS }} - exclude: - - \ No newline at end of file diff --git a/salt/curator/files/curator.yml b/salt/curator/files/curator.yml deleted file mode 100644 index 4ea1dddf7..000000000 --- a/salt/curator/files/curator.yml +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{% from 'vars/globals.map.jinja' import GLOBALS %} -{% if GLOBALS.role in ['so-searchnode', 'so-heavynode'] %} - {%- set elasticsearch = GLOBALS.node_ip -%} -{% elif GLOBALS.role in ['so-eval', 'so-managersearch', 'so-standalone', 'so-manager'] %} - {%- set elasticsearch = GLOBALS.manager_ip -%} -{%- endif %} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} - ---- -# Remember, leave a key empty if there is no value. None will be a string, -# not a Python "NoneType" -elasticsearch: - client: - hosts: - - https://{{elasticsearch}}:9200 - cloud_id: - ca_certs: - client_cert: - client_key: - verify_certs: False - request_timeout: 30 - other_settings: - api_key: - id: - api_key: - master_only: False - username: "{{ ES_USER }}" - password: "{{ ES_PASS }}" - -logging: - loglevel: INFO - logfile: '/var/log/curator/curator.log' - logformat: default - blacklist: ['elasticsearch', 'urllib3'] diff --git a/salt/curator/init.sls b/salt/curator/init.sls deleted file mode 100644 index 201195b60..000000000 --- a/salt/curator/init.sls +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{% from 'curator/map.jinja' import CURATORMERGED %} - -include: -{% if CURATORMERGED.enabled %} - - curator.enabled -{% else %} - - curator.disabled -{% endif %} diff --git a/salt/curator/map.jinja b/salt/curator/map.jinja deleted file mode 100644 index 517209635..000000000 --- a/salt/curator/map.jinja +++ /dev/null @@ -1,7 +0,0 @@ -{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one - or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at - https://securityonion.net/license; you may not use this file except in compliance with the - Elastic License 2.0. #} - -{% import_yaml 'curator/defaults.yaml' as CURATORDEFAULTS %} -{% set CURATORMERGED = salt['pillar.get']('curator', CURATORDEFAULTS.curator, merge=true) %} diff --git a/salt/curator/soc_curator.yaml b/salt/curator/soc_curator.yaml deleted file mode 100644 index acfba7f85..000000000 --- a/salt/curator/soc_curator.yaml +++ /dev/null @@ -1,39 +0,0 @@ -curator: - enabled: - description: You can enable or disable Curator. - helpLink: curator.html - elasticsearch: - index_settings: - so-beats: - delete: &delete - description: Age, in days, when Curator deletes the index. - helpLink: curator.html - forcedType: int - so-elasticsearch: - delete: *delete - so-firewall: - delete: *delete - so-ids: - delete: *delete - so-import: - delete: *delete - so-kratos: - delete: *delete - so-kibana: - delete: *delete - so-logstash: - delete: *delete - so-netflow: - delete: *delete - so-osquery: - delete: *delete - so-ossec: - delete: *delete - so-redis: - delete: *delete - so-strelka: - delete: *delete - so-syslog: - delete: *delete - so-zeek: - delete: *delete diff --git a/salt/curator/sostatus.sls b/salt/curator/sostatus.sls deleted file mode 100644 index de6459a6d..000000000 --- a/salt/curator/sostatus.sls +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{% from 'allowed_states.map.jinja' import allowed_states %} -{% if sls.split('.')[0] in allowed_states %} - -append_so-curator_so-status.conf: - file.append: - - name: /opt/so/conf/so-status/so-status.conf - - text: so-curator - - unless: grep -q so-curator /opt/so/conf/so-status/so-status.conf - -{% else %} - -{{sls}}_state_not_allowed: - test.fail_without_changes: - - name: {{sls}}_state_not_allowed - -{% endif %} diff --git a/salt/curator/tools/sbin/so-curator-delete b/salt/curator/tools/sbin/so-curator-delete deleted file mode 100644 index 2d128bfdf..000000000 --- a/salt/curator/tools/sbin/so-curator-delete +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - - -APP=delete -lf=/tmp/$APP-pidLockFile -# create empty lock file if none exists -cat /dev/null >> $lf -read lastPID < $lf -# if lastPID is not null and a process with that pid exists , exit -[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit -echo $$ > $lf - -docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/delete.yml > /dev/null 2>&1 diff --git a/salt/curator/tools/sbin/so-curator-restart b/salt/curator/tools/sbin/so-curator-restart deleted file mode 100644 index f57e7b22e..000000000 --- a/salt/curator/tools/sbin/so-curator-restart +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - - - -. /usr/sbin/so-common - -/usr/sbin/so-restart curator $1 diff --git a/salt/curator/tools/sbin/so-curator-start b/salt/curator/tools/sbin/so-curator-start deleted file mode 100644 index c5f0fc4d1..000000000 --- a/salt/curator/tools/sbin/so-curator-start +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - - - -. /usr/sbin/so-common - -/usr/sbin/so-start curator $1 diff --git a/salt/curator/tools/sbin/so-curator-stop b/salt/curator/tools/sbin/so-curator-stop deleted file mode 100644 index 30fb07e4b..000000000 --- a/salt/curator/tools/sbin/so-curator-stop +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - - - -. /usr/sbin/so-common - -/usr/sbin/so-stop curator $1 diff --git a/salt/docker/defaults.yaml b/salt/docker/defaults.yaml index a5d6c5d6d..9a27843ae 100644 --- a/salt/docker/defaults.yaml +++ b/salt/docker/defaults.yaml @@ -159,11 +159,6 @@ docker: custom_bind_mounts: [] extra_hosts: [] extra_env: [] - 'so-curator': - final_octet: 43 - custom_bind_mounts: [] - extra_hosts: [] - extra_env: [] 'so-elastic-fleet-package-registry': final_octet: 44 port_bindings: diff --git a/salt/docker/soc_docker.yaml b/salt/docker/soc_docker.yaml index d227a3e85..850324a9e 100644 --- a/salt/docker/soc_docker.yaml +++ b/salt/docker/soc_docker.yaml @@ -60,7 +60,6 @@ docker: so-strelka-gatekeeper: *dockerOptions so-strelka-coordinator: *dockerOptions so-elastalert: *dockerOptions - so-curator: *dockerOptions so-elastic-fleet-package-registry: *dockerOptions so-idh: *dockerOptions so-elastic-agent: *dockerOptions diff --git a/salt/elasticsearch/disabled.sls b/salt/elasticsearch/disabled.sls index 210ad59a4..2453f9f77 100644 --- a/salt/elasticsearch/disabled.sls +++ b/salt/elasticsearch/disabled.sls @@ -18,6 +18,10 @@ so-elasticsearch_so-status.disabled: - name: /opt/so/conf/so-status/so-status.conf - regex: ^so-elasticsearch$ +so-elasticsearch-indices-delete: + cron.absent: + - identifier: so-elasticsearch-indices-delete + {% else %} {{sls}}_state_not_allowed: diff --git a/salt/elasticsearch/enabled.sls b/salt/elasticsearch/enabled.sls index f7ab7749f..349fe64c4 100644 --- a/salt/elasticsearch/enabled.sls +++ b/salt/elasticsearch/enabled.sls @@ -195,6 +195,26 @@ so-elasticsearch-roles-load: - require: - docker_container: so-elasticsearch - file: elasticsearch_sbin_jinja +{% if grains.role in ['so-eval', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-manager'] %} +so-curator-cluster-close: + cron.absent: + - identifier: so-curator-cluster-close + +so-curator-cluster-delete: + cron.absent: + - identifier: so-curator-cluster-delete + +so-elasticsearch-indices-delete: + cron.present: + - name: /usr/sbin/so-elasticsearch-indices-delete > /opt/so/log/curator/cron-elasticsearch-indices-delete.log 2>&1 + - identifier: so-elasticsearch-indices-delete + - user: root + - minute: '*/5' + - hour: '*' + - daymonth: '*' + - month: '*' + - dayweek: '*' +{% endif %} {% endif %} {% else %} diff --git a/salt/curator/tools/sbin/so-curator-cluster-delete b/salt/elasticsearch/tools/sbin/so-elasticsearch-indices-delete similarity index 92% rename from salt/curator/tools/sbin/so-curator-cluster-delete rename to salt/elasticsearch/tools/sbin/so-elasticsearch-indices-delete index 0f7945b78..036ff844f 100755 --- a/salt/curator/tools/sbin/so-curator-cluster-delete +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-indices-delete @@ -14,4 +14,4 @@ read lastPID < $lf [ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit echo $$ > $lf -/usr/sbin/so-curator-cluster-delete-delete +/usr/sbin/so-elasticsearch-indices-delete-delete diff --git a/salt/curator/tools/sbin_jinja/so-curator-cluster-delete-delete b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-indices-delete-delete similarity index 96% rename from salt/curator/tools/sbin_jinja/so-curator-cluster-delete-delete rename to salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-indices-delete-delete index e0c5144bc..292265095 100755 --- a/salt/curator/tools/sbin_jinja/so-curator-cluster-delete-delete +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-indices-delete-delete @@ -9,8 +9,8 @@ {%- set ELASTICSEARCH_HOST = GLOBALS.node_ip -%} {%- set RETENTION = salt['pillar.get']('elasticsearch:retention', ELASTICDEFAULTS.elasticsearch.retention, merge=true) -%} -LOG="/opt/so/log/curator/so-curator-cluster-delete.log" -ALERT_LOG="/opt/so/log/curator/alert.log" +LOG="/opt/so/log/elasticsearch/so-elasticsearch-indices-delete.log" +ALERT_LOG="/opt/so/log/elasticsearch/indices-delete-alert.log" LOG_SIZE_LIMIT_GB=$(/usr/sbin/so-elasticsearch-cluster-space-total {{ RETENTION.retention_pct}}) LOG_SIZE_LIMIT=$(( "$LOG_SIZE_LIMIT_GB" * 1000 * 1000 * 1000 )) ITERATION=0 diff --git a/salt/firewall/containers.map.jinja b/salt/firewall/containers.map.jinja index 02e8a4644..0ba2389e9 100644 --- a/salt/firewall/containers.map.jinja +++ b/salt/firewall/containers.map.jinja @@ -2,7 +2,6 @@ {% if GLOBALS.role == 'so-eval' %} {% set NODE_CONTAINERS = [ - 'so-curator', 'so-dockerregistry', 'so-elasticsearch', 'so-elastic-fleet', @@ -25,7 +24,6 @@ {% elif GLOBALS.role == 'so-manager' or GLOBALS.role == 'so-standalone' or GLOBALS.role == 'so-managersearch' %} {% set NODE_CONTAINERS = [ - 'so-curator', 'so-dockerregistry', 'so-elasticsearch', 'so-elastic-fleet', @@ -56,7 +54,6 @@ {% elif GLOBALS.role == 'so-heavynode' %} {% set NODE_CONTAINERS = [ - 'so-curator', 'so-elasticsearch', 'so-elastic-agent', 'so-logstash', diff --git a/salt/logrotate/defaults.yaml b/salt/logrotate/defaults.yaml index 4d6a688e4..7333c78e9 100644 --- a/salt/logrotate/defaults.yaml +++ b/salt/logrotate/defaults.yaml @@ -80,7 +80,7 @@ logrotate: - extension .log - dateext - dateyesterday - /opt/so/log/curator/*_x_log: + /opt/so/log/elasticsearch/*indices-delete*_x_log: - daily - rotate 14 - missingok diff --git a/salt/logrotate/soc_logrotate.yaml b/salt/logrotate/soc_logrotate.yaml index 5e6c78fcc..62aa935c9 100644 --- a/salt/logrotate/soc_logrotate.yaml +++ b/salt/logrotate/soc_logrotate.yaml @@ -56,13 +56,6 @@ logrotate: multiline: True global: True forcedType: "[]string" - "/opt/so/log/curator/*_x_log": - description: List of logrotate options for this file. - title: /opt/so/log/curator/*.log - advanced: True - multiline: True - global: True - forcedType: "[]string" "/opt/so/log/elasticfleet/*_x_log": description: List of logrotate options for this file. title: /opt/so/log/elasticfleet/*.log @@ -77,6 +70,13 @@ logrotate: multiline: True global: True forcedType: "[]string" + "/opt/so/log/elasticsearch/*indices-delete*_x_log": + description: List of logrotate options for this file. + title: /opt/so/log/elasticsearch/*indices-delete.log + advanced: True + multiline: True + global: True + forcedType: "[]string" "/opt/so/log/suricata/*_x_log": description: List of logrotate options for this file. title: /opt/so/log/suricata/*.log diff --git a/salt/manager/tools/sbin/so-elastic-auth-password-reset b/salt/manager/tools/sbin/so-elastic-auth-password-reset index 5bc5230c3..b6bfd8721 100755 --- a/salt/manager/tools/sbin/so-elastic-auth-password-reset +++ b/salt/manager/tools/sbin/so-elastic-auth-password-reset @@ -97,8 +97,6 @@ function soUserSync() { salt -C 'G@role:so-standalone or G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managersearch or G@role:so-searchnode or G@role:so-heavynode' state.apply logstash queue=True printf "\nApplying kibana state to the appropriate nodes.\n\n" salt -C 'G@role:so-standalone or G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managersearch' state.apply kibana queue=True - printf "\nApplying curator state to the appropriate nodes.\n\n" - salt -C 'G@role:so-standalone or G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managersearch or G@role:so-searchnode or G@role:so-heavynode' state.apply curator queue=True } function highstateManager() { diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index ada56dc2a..d5225cc82 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -302,13 +302,6 @@ function add_strelka_to_minion() { " " >> $PILLARFILE } -function add_curator_to_minion() { - printf '%s\n'\ - "curator:"\ - " enabled: True"\ - " " >> $PILLARFILE -} - function add_telegraf_to_minion() { printf '%s\n'\ "telegraf:"\ @@ -435,7 +428,6 @@ function createEVAL() { add_playbook_to_minion add_elastalert_to_minion add_kibana_to_minion - add_curator_to_minion add_telegraf_to_minion add_influxdb_to_minion add_nginx_to_minion @@ -458,7 +450,6 @@ function createSTANDALONE() { add_elastalert_to_minion add_kibana_to_minion add_redis_to_minion - add_curator_to_minion add_telegraf_to_minion add_influxdb_to_minion add_nginx_to_minion @@ -478,7 +469,6 @@ function createMANAGER() { add_elastalert_to_minion add_kibana_to_minion add_redis_to_minion - add_curator_to_minion add_telegraf_to_minion add_influxdb_to_minion add_nginx_to_minion @@ -498,7 +488,6 @@ function createMANAGERSEARCH() { add_elastalert_to_minion add_kibana_to_minion add_redis_to_minion - add_curator_to_minion add_telegraf_to_minion add_influxdb_to_minion add_nginx_to_minion @@ -548,7 +537,6 @@ function createHEAVYNODE() { add_sensor_to_minion add_strelka_to_minion add_redis_to_minion - add_curator_to_minion add_telegraf_to_minion } @@ -679,4 +667,4 @@ case "$OPERATION" in *) usage ;; -esac \ No newline at end of file +esac diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 35c934772..2485900c5 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -465,9 +465,8 @@ post_to_2.4.30() { } post_to_2.4.40() { - echo "Removing Curator close and delete files" - rm -f /opt/so/conf/curator/action/*-close.y*ml - rm -f /opt/so/conf/curator/action/logs*-delete.y*ml + echo "Removing Curator configuration" + rm -rf /opt/so/conf/curator/ } repo_sync() { diff --git a/salt/top.sls b/salt/top.sls index 4f84e17ac..5e7d13532 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -74,7 +74,6 @@ base: - suricata - zeek - strelka - - curator - elastalert - utility - soctopus @@ -104,7 +103,6 @@ base: - redis - elastic-fleet-package-registry - kibana - - curator - elastalert - utility - soctopus @@ -140,7 +138,6 @@ base: - suricata - zeek - strelka - - curator - elastalert - utility - soctopus @@ -179,7 +176,6 @@ base: - elasticsearch - logstash - redis - - curator - elastic-fleet-package-registry - kibana - elastalert @@ -199,7 +195,6 @@ base: - elasticsearch - logstash - redis - - curator - strelka - pcap - suricata diff --git a/setup/so-functions b/setup/so-functions index 6ad0947d1..e1faa275e 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1412,7 +1412,7 @@ make_some_dirs() { mkdir -p $local_salt_dir/salt/firewall/portgroups mkdir -p $local_salt_dir/salt/firewall/ports - for THEDIR in bpf pcap elasticsearch ntp firewall redis backup influxdb strelka sensoroni curator soc soctopus docker zeek suricata nginx telegraf logstash soc manager kratos idstools idh elastalert global;do + for THEDIR in bpf pcap elasticsearch ntp firewall redis backup influxdb strelka sensoroni soc soctopus docker zeek suricata nginx telegraf logstash soc manager kratos idstools idh elastalert global;do mkdir -p $local_salt_dir/pillar/$THEDIR touch $local_salt_dir/pillar/$THEDIR/adv_$THEDIR.sls touch $local_salt_dir/pillar/$THEDIR/soc_$THEDIR.sls @@ -1555,7 +1555,6 @@ reserve_group_ids() { logCmd "groupadd -g 931 logstash" logCmd "groupadd -g 932 kibana" logCmd "groupadd -g 933 elastalert" - logCmd "groupadd -g 934 curator" logCmd "groupadd -g 937 zeek" logCmd "groupadd -g 940 suricata" logCmd "groupadd -g 941 stenographer" diff --git a/setup/so-variables b/setup/so-variables index 7f6522487..4d3cbb739 100644 --- a/setup/so-variables +++ b/setup/so-variables @@ -1,4 +1,4 @@ -#!/bin/bash +a!/bin/bash SOVERSION=$(cat ../VERSION) mkdir -p /nsm @@ -112,12 +112,6 @@ export sensoroni_pillar_file adv_sensoroni_pillar_file="$local_salt_dir/pillar/sensoroni/adv_sensoroni.sls" export adv_sensoroni_pillar_file -curator_pillar_file="$local_salt_dir/pillar/curator/soc_curator.sls" -export curator_pillar_file - -adv_curator_pillar_file="$local_salt_dir/pillar/curator/adv_curator.sls" -export adv_curator_pillar_file - soctopus_pillar_file="$local_salt_dir/pillar/soctopus/soc_soctopus.sls" export soctopus_pillar_file From f7373ed79c241ea62bf16a070daaa18eb202713f Mon Sep 17 00:00:00 2001 From: Wes Date: Mon, 11 Dec 2023 19:20:52 +0000 Subject: [PATCH 03/21] Stop Curator, remove scripts and status --- salt/manager/tools/sbin/soup | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 2485900c5..4a0adbc98 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -465,8 +465,12 @@ post_to_2.4.30() { } post_to_2.4.40() { + echo "Stopping Curator" + so-curator-stop echo "Removing Curator configuration" rm -rf /opt/so/conf/curator/ + rm -f /usr/sbin/so-curator-* + sed -i '/so-curator/d' /opt/so/conf/so-status/so-status.conf } repo_sync() { From 25c39540c8c7add87d86a4c8964f17c9272d2bae Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 11 Dec 2023 14:48:46 -0500 Subject: [PATCH 04/21] fix import stats --- salt/common/tools/sbin/so-log-check | 1 + salt/telegraf/defaults.yaml | 1 + salt/telegraf/etc/telegraf.conf | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check index d2582ff94..282411ecc 100755 --- a/salt/common/tools/sbin/so-log-check +++ b/salt/common/tools/sbin/so-log-check @@ -144,6 +144,7 @@ if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|status 200" # false positive (request successful, contained error string in content) EXCLUDED_ERRORS="$EXCLUDED_ERRORS|app_layer.error" # false positive (suricata 7) in stats.log e.g. app_layer.error.imap.parser | Total | 0 EXCLUDED_ERRORS="$EXCLUDED_ERRORS|is not an ip string literal" # false positive (Open Canary logging out blank IP addresses) + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|cannot join on an empty table" # false positive (InfluxDB flux query, import nodes) fi if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then diff --git a/salt/telegraf/defaults.yaml b/salt/telegraf/defaults.yaml index 676f365cf..fa9d9b2b9 100644 --- a/salt/telegraf/defaults.yaml +++ b/salt/telegraf/defaults.yaml @@ -48,6 +48,7 @@ telegraf: - redis.sh - sostatus.sh import: + - influxdbsize.sh - os.sh - sostatus.sh sensor: diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf index 45b1283e0..1c5801645 100644 --- a/salt/telegraf/etc/telegraf.conf +++ b/salt/telegraf/etc/telegraf.conf @@ -186,7 +186,7 @@ # # # Read stats from one or more Elasticsearch servers or clusters -{%- if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone', 'so-heavynode'] %} +{%- if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-import'] %} [[inputs.elasticsearch]] servers = ["https://{{ NODEIP }}:9200"] cluster_stats = true From b1721b646749d4b390ed5018e5e2dbfcb9586022 Mon Sep 17 00:00:00 2001 From: Wes Date: Mon, 11 Dec 2023 21:43:25 +0000 Subject: [PATCH 05/21] Fix directory --- salt/elasticsearch/enabled.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/enabled.sls b/salt/elasticsearch/enabled.sls index 349fe64c4..6fe4635a3 100644 --- a/salt/elasticsearch/enabled.sls +++ b/salt/elasticsearch/enabled.sls @@ -206,7 +206,7 @@ so-curator-cluster-delete: so-elasticsearch-indices-delete: cron.present: - - name: /usr/sbin/so-elasticsearch-indices-delete > /opt/so/log/curator/cron-elasticsearch-indices-delete.log 2>&1 + - name: /usr/sbin/so-elasticsearch-indices-delete > /opt/so/log/elasticsearch/cron-elasticsearch-indices-delete.log 2>&1 - identifier: so-elasticsearch-indices-delete - user: root - minute: '*/5' From 54c3167b104f2c8d2694b370437b6665055ebe2f Mon Sep 17 00:00:00 2001 From: Wes Date: Tue, 12 Dec 2023 05:25:50 +0000 Subject: [PATCH 06/21] Delete data streams when necessary --- .../tools/sbin_jinja/so-elasticsearch-indices-delete-delete | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-indices-delete-delete b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-indices-delete-delete index 292265095..07feb36bd 100755 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-indices-delete-delete +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-indices-delete-delete @@ -51,8 +51,11 @@ while overlimit && [[ $ITERATION -lt $MAX_ITERATIONS ]]; do if [ "${INDEX}" != "${CURRENT_WRITE_INDEX}" ]; then # This should not be a write index, so we should be allowed to delete it printf "\n$(date) - Used disk space exceeds LOG_SIZE_LIMIT (${LOG_SIZE_LIMIT_GB} GB) - Deleting ${INDEX} index...\n" >> ${LOG} - /usr/sbin/so-elasticsearch-query ${INDEX} -XDELETE >> ${LOG} 2>&1 + /usr/sbin/so-elasticsearch-query ${INDEX} -XDELETE >> ${LOG} 2>&1 fi + else + printf "\n$(date) - Used disk space exceeds LOG_SIZE_LIMIT (${LOG_SIZE_LIMIT_GB} GB) - There is only one backing index (${INDEX}). Deleting ${DATASTREAM} data stream...\n" >> ${LOG} + /usr/sbin/so-elasticsearch-query _data_stream/$DATASTREAM -XDELETE >> ${LOG} 2>&1 fi if ! overlimit ; then exit From bbe091fa1433b05a5cc744232e5281a4e80bec24 Mon Sep 17 00:00:00 2001 From: Wes Date: Tue, 12 Dec 2023 15:08:47 +0000 Subject: [PATCH 07/21] Fix accidental change --- setup/so-variables | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-variables b/setup/so-variables index 4d3cbb739..511dfc43b 100644 --- a/setup/so-variables +++ b/setup/so-variables @@ -1,4 +1,4 @@ -a!/bin/bash +#!/bin/bash SOVERSION=$(cat ../VERSION) mkdir -p /nsm From aaf60bea87dc1623297d461279d8d6baa7876afb Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Tue, 12 Dec 2023 16:30:17 -0500 Subject: [PATCH 08/21] Update so-nsm-clear --- salt/common/tools/sbin/so-nsm-clear | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/salt/common/tools/sbin/so-nsm-clear b/salt/common/tools/sbin/so-nsm-clear index 3d9596238..c9e5e86d2 100755 --- a/salt/common/tools/sbin/so-nsm-clear +++ b/salt/common/tools/sbin/so-nsm-clear @@ -41,8 +41,13 @@ done if [ $SKIP -ne 1 ]; then # Inform user we are about to delete all data echo - echo "This script will delete all NIDS data (PCAP, Suricata, Zeek)" - echo "If you would like to proceed, please type "AGREE" and hit ENTER." + echo "This script will delete all NSM data from /nsm." + echo + echo "This includes Suricata data, Zeek data, and full packet capture (PCAP)." + echo + echo "This will NOT delete any Suricata or Zeek logs that have already been ingested into Elasticsearch." + echo + echo "If you would like to proceed, then type AGREE and press ENTER." echo # Read user input read INPUT @@ -54,8 +59,8 @@ delete_pcap() { [ -d $PCAP_DATA ] && so-pcap-stop && rm -rf $PCAP_DATA/* && so-pcap-start } delete_suricata() { - SURI_LOG="/opt/so/log/suricata/eve.json" - [ -f $SURI_LOG ] && so-suricata-stop && rm -f $SURI_LOG && so-suricata-start + SURI_LOG="/nsm/suricata/" + [ -d $SURI_LOG ] && so-suricata-stop && rm -rf $SURI_LOG/* && so-suricata-start } delete_zeek() { ZEEK_LOG="/nsm/zeek/logs/" From d49d13289e6109a5f1e02342ab38a301c5fafb62 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Tue, 12 Dec 2023 16:37:06 -0500 Subject: [PATCH 09/21] Update so-elastic-clear --- salt/elasticsearch/tools/sbin/so-elastic-clear | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/tools/sbin/so-elastic-clear b/salt/elasticsearch/tools/sbin/so-elastic-clear index 085327fc4..af7b91ce2 100755 --- a/salt/elasticsearch/tools/sbin/so-elastic-clear +++ b/salt/elasticsearch/tools/sbin/so-elastic-clear @@ -67,7 +67,7 @@ if [ $SKIP -ne 1 ]; then echo echo "This script will delete all data (documents, indices, etc.) in the Elasticsearch database." echo - echo "If you would like to proceed, please type "AGREE" and hit ENTER." + echo "If you would like to proceed, then type AGREE and press ENTER." echo # Read user input read INPUT From 2c4d0a0d7107660f04726b14a9941a2112c76519 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Tue, 12 Dec 2023 16:37:50 -0500 Subject: [PATCH 10/21] Update so-elastic-fleet-reset --- salt/manager/tools/sbin_jinja/so-elastic-fleet-reset | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/manager/tools/sbin_jinja/so-elastic-fleet-reset b/salt/manager/tools/sbin_jinja/so-elastic-fleet-reset index 564156af9..f1112c723 100644 --- a/salt/manager/tools/sbin_jinja/so-elastic-fleet-reset +++ b/salt/manager/tools/sbin_jinja/so-elastic-fleet-reset @@ -18,7 +18,7 @@ echo "This includes data previously ingested with Fleet such as Zeek and Suricat echo "Deployed Elastic Agents will no longer be enrolled and will need to be reinstalled." echo "This script should only be used as a last resort to reinstall Elastic Fleet." echo -echo "If you would like to proceed, type AGREE and hit ENTER." +echo "If you would like to proceed, then type AGREE and press ENTER." echo # Read user input read INPUT @@ -93,4 +93,4 @@ so-elastic-fleet-setup status "Re-installing Elastic Agent on all Grid Nodes..." salt \* state.apply elasticfleet.install_agent_grid queue=True -status "Elastic Fleet Reset complete...." \ No newline at end of file +status "Elastic Fleet Reset complete...." From 03279732b7f44aaf238bd1f9e474793cab87a318 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 13 Dec 2023 10:09:36 -0500 Subject: [PATCH 11/21] upgrade cla action --- .github/workflows/contrib.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/contrib.yml b/.github/workflows/contrib.yml index 1cb3b773b..395675b43 100644 --- a/.github/workflows/contrib.yml +++ b/.github/workflows/contrib.yml @@ -11,7 +11,7 @@ jobs: steps: - name: "Contributor Check" if: (github.event.comment.body == 'recheck' || github.event.comment.body == 'I have read the CLA Document and I hereby sign the CLA') || github.event_name == 'pull_request_target' - uses: cla-assistant/github-action@v2.1.3-beta + uses: cla-assistant/github-action@v2.3.1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} PERSONAL_ACCESS_TOKEN : ${{ secrets.PERSONAL_ACCESS_TOKEN }} From bd26a52227a5bad5c0c3a0fceae11258e9e8046b Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 13 Dec 2023 10:10:23 -0500 Subject: [PATCH 12/21] upgrade cla action --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 7f2e97617..29630cd6d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4.0-kilo +2.4.40 From c0ab8f24e9f73737523e71db4bd0c49e3ccefc42 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 13 Dec 2023 10:10:51 -0500 Subject: [PATCH 13/21] upgrade cla action --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 29630cd6d..8ea99f559 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4.40 +2.4.30 From b0a69d30c994d45f46f91e9ac70ecf1798610c7c Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 13 Dec 2023 10:44:03 -0500 Subject: [PATCH 14/21] Add Anomali, Cybersixgill, Snort, and ThreatQuotient packages --- salt/elasticfleet/defaults.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/elasticfleet/defaults.yaml b/salt/elasticfleet/defaults.yaml index 7d3883895..2c0fcb816 100644 --- a/salt/elasticfleet/defaults.yaml +++ b/salt/elasticfleet/defaults.yaml @@ -76,6 +76,7 @@ elasticfleet: - pulse_connect_secure - redis - sentinel_one + - snort - snyk - sonicwall_firewall - sophos @@ -85,9 +86,12 @@ elasticfleet: - tcp - tenable_sc - ti_abusech + - ti_anomali + - ti_cybersixgill - ti_misp - ti_otx - ti_recordedfuture + - ti_threatq - udp - vsphere - windows From cdac2bfa160f44cb7c8a84ff6390a77f5f0da147 Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 13 Dec 2023 11:03:25 -0500 Subject: [PATCH 15/21] Add Anomali, Cybersixgill, Snort, and ThreatQuotient --- salt/elasticsearch/defaults.yaml | 176 +++++++++++++++++++++++++++++++ 1 file changed, 176 insertions(+) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 02c2529a6..66916acd1 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -6737,6 +6737,50 @@ elasticsearch: set_priority: priority: 50 min_age: 30d + so-logs-snort_x_log: + index_sorting: False + index_template: + index_patterns: + - "logs-snort.log-*" + template: + settings: + index: + lifecycle: + name: so-logs-snort.log-logs + number_of_replicas: 0 + composed_of: + - "logs-snort.log@package" + - "logs-snort.log@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d so-logs-snyk_x_audit: index_sorting: false index_template: @@ -7622,6 +7666,94 @@ elasticsearch: set_priority: priority: 50 min_age: 30d + so-logs-ti_anomali_x_threatstream: + index_sorting: False + index_template: + index_patterns: + - "logs-ti_anomali.threatstream-*" + template: + settings: + index: + lifecycle: + name: so-logs-ti_anomali.threatstream-logs + number_of_replicas: 0 + composed_of: + - "logs-ti_anomali.threatstream@package" + - "logs-ti_anomali.threatstream@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-ti_cybersixgill_x_threat: + index_sorting: False + index_template: + index_patterns: + - "logs-ti_cybersixgill.threat-*" + template: + settings: + index: + lifecycle: + name: so-logs-ti_cybersixgill.threat-logs + number_of_replicas: 0 + composed_of: + - "logs-ti_cybersixgill.threat@package" + - "logs-ti_cybersixgill.threat@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d so-logs-ti_misp_x_threat: index_sorting: false index_template: @@ -7842,6 +7974,50 @@ elasticsearch: set_priority: priority: 50 min_age: 30d + so-logs-ti_threatq_x_threat: + index_sorting: False + index_template: + index_patterns: + - "logs-ti_threatq.threat-*" + template: + settings: + index: + lifecycle: + name: so-logs-ti_threatq.threat-logs + number_of_replicas: 0 + composed_of: + - "logs-ti_threatq.threat@package" + - "logs-ti_threatq.threat@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d so-logs-vsphere_x_log: index_sorting: False index_template: From 8cf5d9c1a6dacb322004dc673242c7aca1e52feb Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 13 Dec 2023 11:55:40 -0500 Subject: [PATCH 16/21] Annotations --- salt/elasticsearch/soc_elasticsearch.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index 9d9b3d294..0b93a6c1a 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -407,16 +407,20 @@ elasticsearch: so-logs-sentinel_one_x_group: *indexSettings so-logs-sentinel_one_x_threat: *indexSettings so-logs-sonicwall_firewall_x_log: *indexSettings + so-logs-snort_x_log: *indexSettings so-logs-symantec_endpoint_x_log: *indexSettings so-logs-ti_abusech_x_malware: *indexSettings so-logs-ti_abusech_x_malwarebazaar: *indexSettings so-logs-ti_abusech_x_threatfox: *indexSettings so-logs-ti_abusech_x_url: *indexSettings + so-logs-ti_anomali_x_threatstream: *indexSettings + so-logs-ti_cybersixgill_x_threat: *indexSettings so-logs-ti_misp_x_threat: *indexSettings so-logs-ti_misp_x_threat_attributes: *indexSettings so-logs-ti_otx_x_threat: *indexSettings so-logs-ti_recordedfuture_x_latest_ioc-template: *indexSettings so-logs-ti_recordedfuture_x_threat: *indexSettings + so-logs-ti_threatq_x_threat: *indexSettings so-logs-zscaler_zia_x_alerts: *indexSettings so-logs-zscaler_zia_x_dns: *indexSettings so-logs-zscaler_zia_x_firewall: *indexSettings From 997d3237635177a1d4adec42de986da3d85d77ee Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 14 Dec 2023 08:55:18 -0500 Subject: [PATCH 17/21] more log false alarms --- setup/so-verify | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/setup/so-verify b/setup/so-verify index 3c20d22b4..6f47940ac 100755 --- a/setup/so-verify +++ b/setup/so-verify @@ -33,13 +33,17 @@ log_has_errors() { # Ignore Failed: 0 since that is the salt state output, and we detect state failures # via Result: False already. - # This is ignored for Ubuntu + # This is ignored for Ubuntu: # Failed to restart snapd.mounts-pre.target: Operation refused, unit snapd.mounts-pre.target # may be requested by dependency only (it is configured to refuse manual start/stop). # Command failed with exit code is output during retry loops. # "remove failed" is caused by a warning generated by upgrade of libwbclient + + # Exit code 100 failure is likely apt-get running in the background, we wait for it to unlock. + + # Failed to deduce dest mapping appears to occur when a shard isn't yet ready. Temporary. grep -E "FAILED|Failed|failed|ERROR|Result: False|Error is not recoverable" "$setup_log" | \ grep -vE "The Salt Master has cached the public key for this node" | \ @@ -58,6 +62,7 @@ log_has_errors() { grep -vE "remove failed" | \ grep -vE "Failed to restart snapd" | \ grep -vE "Login Failed Details" | \ + grep -vE "Failed to deduce dest mappings" | \ grep -vE "response from daemon: unauthorized" | \ grep -vE "Reading first line of patchfile" | \ grep -vE "Command failed with exit code" | \ From 3dbf97944d2550789193484d390553fdab147332 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 14 Dec 2023 10:26:29 -0500 Subject: [PATCH 18/21] fix extra_hosts. https://github.com/Security-Onion-Solutions/securityonion/issues/12015 --- salt/elasticsearch/enabled.sls | 7 ++++++- salt/logstash/enabled.sls | 11 ++++++++--- salt/soc/enabled.sls | 23 ++++++++++++++--------- 3 files changed, 28 insertions(+), 13 deletions(-) diff --git a/salt/elasticsearch/enabled.sls b/salt/elasticsearch/enabled.sls index f7ab7749f..591840817 100644 --- a/salt/elasticsearch/enabled.sls +++ b/salt/elasticsearch/enabled.sls @@ -26,7 +26,12 @@ so-elasticsearch: - networks: - sobridge: - ipv4_address: {{ DOCKER.containers['so-elasticsearch'].ip }} - - extra_hosts: {{ LOGSTASH_NODES }} + - extra_hosts: + {% for node in LOGSTASH_NODES %} + {% for hostname, ip in node.items() %} + - {{hostname}}:{{ip}} + {% endfor %} + {% endfor %} {% if DOCKER.containers['so-elasticsearch'].extra_hosts %} {% for XTRAHOST in DOCKER.containers['so-elasticsearch'].extra_hosts %} - {{ XTRAHOST }} diff --git a/salt/logstash/enabled.sls b/salt/logstash/enabled.sls index c76f81d21..5337351fc 100644 --- a/salt/logstash/enabled.sls +++ b/salt/logstash/enabled.sls @@ -33,11 +33,16 @@ so-logstash: - sobridge: - ipv4_address: {{ DOCKER.containers['so-logstash'].ip }} - user: logstash - - extra_hosts: {{ REDIS_NODES }} + - extra_hosts: + {% for node in REDIS_NODES %} + {% for hostname, ip in node.items() %} + - {{hostname}}:{{ip}} + {% endfor %} + {% endfor %} {% if DOCKER.containers['so-logstash'].extra_hosts %} - {% for XTRAHOST in DOCKER.containers['so-logstash'].extra_hosts %} + {% for XTRAHOST in DOCKER.containers['so-logstash'].extra_hosts %} - {{ XTRAHOST }} - {% endfor %} + {% endfor %} {% endif %} - environment: - LS_JAVA_OPTS=-Xms{{ lsheap }} -Xmx{{ lsheap }} diff --git a/salt/soc/enabled.sls b/salt/soc/enabled.sls index 9e588e5b1..2661587f4 100644 --- a/salt/soc/enabled.sls +++ b/salt/soc/enabled.sls @@ -33,21 +33,26 @@ so-soc: - /opt/so/conf/soc/soc_users_roles:/opt/sensoroni/rbac/users_roles:rw - /opt/so/conf/soc/queue:/opt/sensoroni/queue:rw - /opt/so/saltstack:/opt/so/saltstack:rw - - extra_hosts: {{ DOCKER_EXTRA_HOSTS }} - {% if DOCKER.containers['so-soc'].extra_hosts %} - {% for XTRAHOST in DOCKER.containers['so-soc'].extra_hosts %} + - extra_hosts: + {% for node in DOCKER_EXTRA_HOSTS %} + {% for hostname, ip in node.items() %} + - {{hostname}}:{{ip}} + {% endfor %} + {% endfor %} + {% if DOCKER.containers['so-soc'].extra_hosts %} + {% for XTRAHOST in DOCKER.containers['so-soc'].extra_hosts %} - {{ XTRAHOST }} - {% endfor %} - {% endif %} + {% endfor %} + {% endif %} - port_bindings: - {% for BINDING in DOCKER.containers['so-soc'].port_bindings %} + {% for BINDING in DOCKER.containers['so-soc'].port_bindings %} - {{ BINDING }} - {% endfor %} + {% endfor %} {% if DOCKER.containers['so-soc'].extra_env %} - environment: - {% for XTRAENV in DOCKER.containers['so-soc'].extra_env %} + {% for XTRAENV in DOCKER.containers['so-soc'].extra_env %} - {{ XTRAENV }} - {% endfor %} + {% endfor %} {% endif %} - watch: - file: /opt/so/conf/soc/* From 22fcccef1c299915730afdb1af2ca9ea0c8d06b8 Mon Sep 17 00:00:00 2001 From: Wes Date: Thu, 14 Dec 2023 16:53:19 +0000 Subject: [PATCH 19/21] Add force option --- .../grid-nodes_general/import-zeek-logs.json | 3 ++- .../integrations-dynamic/grid-nodes_general/zeek-logs.json | 3 ++- .../files/integrations-optional/sublime_platform.json | 3 ++- .../files/integrations/endpoints-initial/osquery.json | 3 ++- .../files/integrations/endpoints-initial/system-endpoints.json | 3 ++- .../integrations/endpoints-initial/windows-endpoints.json | 3 ++- .../integrations/grid-nodes_general/elasticsearch-logs.json | 3 ++- .../files/integrations/grid-nodes_general/idh-logs.json | 3 ++- .../integrations/grid-nodes_general/import-evtx-logs.json | 3 ++- .../integrations/grid-nodes_general/import-suricata-logs.json | 3 ++- .../files/integrations/grid-nodes_general/kratos-logs.json | 3 ++- .../integrations/grid-nodes_general/osquery-grid-nodes.json | 3 ++- .../files/integrations/grid-nodes_general/redis-logs.json | 3 ++- .../integrations/grid-nodes_general/soc-auth-sync-logs.json | 3 ++- .../integrations/grid-nodes_general/soc-salt-relay-logs.json | 3 ++- .../integrations/grid-nodes_general/soc-sensoroni-logs.json | 3 ++- .../files/integrations/grid-nodes_general/soc-server-logs.json | 3 ++- .../files/integrations/grid-nodes_general/strelka-logs.json | 3 ++- .../files/integrations/grid-nodes_general/suricata-logs.json | 3 ++- .../files/integrations/grid-nodes_general/syslog-tcp-514.json | 3 ++- .../files/integrations/grid-nodes_general/syslog-udp-514.json | 3 ++- .../integrations/grid-nodes_general/system-grid-nodes.json | 3 ++- .../integrations/grid-nodes_heavy/osquery-grid-nodes.json | 3 ++- .../files/integrations/grid-nodes_heavy/system-grid-nodes.json | 3 ++- 24 files changed, 48 insertions(+), 24 deletions(-) diff --git a/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/import-zeek-logs.json b/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/import-zeek-logs.json index 0979f98b6..492db03dc 100644 --- a/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/import-zeek-logs.json +++ b/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/import-zeek-logs.json @@ -27,6 +27,7 @@ } } } - } + }, + "force": true } {%- endraw -%} diff --git a/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/zeek-logs.json b/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/zeek-logs.json index 32bff857b..5462dc861 100644 --- a/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/zeek-logs.json +++ b/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/zeek-logs.json @@ -28,6 +28,7 @@ } } } - } + }, + "force": true } {%- endraw -%} diff --git a/salt/elasticfleet/files/integrations-optional/sublime_platform.json b/salt/elasticfleet/files/integrations-optional/sublime_platform.json index 8feedc879..0116ed936 100644 --- a/salt/elasticfleet/files/integrations-optional/sublime_platform.json +++ b/salt/elasticfleet/files/integrations-optional/sublime_platform.json @@ -39,6 +39,7 @@ } } } - } + }, + "force": true } {%- endraw -%} diff --git a/salt/elasticfleet/files/integrations/endpoints-initial/osquery.json b/salt/elasticfleet/files/integrations/endpoints-initial/osquery.json index f1774b905..b3d6c4609 100644 --- a/salt/elasticfleet/files/integrations/endpoints-initial/osquery.json +++ b/salt/elasticfleet/files/integrations/endpoints-initial/osquery.json @@ -16,5 +16,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/endpoints-initial/system-endpoints.json b/salt/elasticfleet/files/integrations/endpoints-initial/system-endpoints.json index b6a21249a..699c07ec3 100644 --- a/salt/elasticfleet/files/integrations/endpoints-initial/system-endpoints.json +++ b/salt/elasticfleet/files/integrations/endpoints-initial/system-endpoints.json @@ -73,5 +73,6 @@ "system-system/metrics": { "enabled": false } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/endpoints-initial/windows-endpoints.json b/salt/elasticfleet/files/integrations/endpoints-initial/windows-endpoints.json index b17986a53..8e9813741 100644 --- a/salt/elasticfleet/files/integrations/endpoints-initial/windows-endpoints.json +++ b/salt/elasticfleet/files/integrations/endpoints-initial/windows-endpoints.json @@ -63,5 +63,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/elasticsearch-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/elasticsearch-logs.json index 4c22f92ee..a2aaf5e0a 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/elasticsearch-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/elasticsearch-logs.json @@ -102,5 +102,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/idh-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/idh-logs.json index 29cc1a879..db4b1a8f6 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/idh-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/idh-logs.json @@ -25,5 +25,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json index baa8683ae..3b1a50560 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json @@ -28,5 +28,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/import-suricata-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/import-suricata-logs.json index 3b8cffcc1..c9b036e36 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/import-suricata-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/import-suricata-logs.json @@ -25,5 +25,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/kratos-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/kratos-logs.json index b1fb71077..6a67c9c1c 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/kratos-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/kratos-logs.json @@ -25,5 +25,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/osquery-grid-nodes.json b/salt/elasticfleet/files/integrations/grid-nodes_general/osquery-grid-nodes.json index 0349c9fc3..5527607bd 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/osquery-grid-nodes.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/osquery-grid-nodes.json @@ -16,5 +16,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/redis-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/redis-logs.json index 6b9cbffaf..b822421eb 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/redis-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/redis-logs.json @@ -72,5 +72,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-auth-sync-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-auth-sync-logs.json index 3aa740881..aa39c177b 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-auth-sync-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-auth-sync-logs.json @@ -25,5 +25,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-salt-relay-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-salt-relay-logs.json index 840f36f6b..cc92092e9 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-salt-relay-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-salt-relay-logs.json @@ -25,5 +25,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-sensoroni-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-sensoroni-logs.json index 60ee95f45..61ad057f4 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-sensoroni-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-sensoroni-logs.json @@ -25,5 +25,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-server-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-server-logs.json index b789adc1d..a875e4bfc 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-server-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-server-logs.json @@ -25,5 +25,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/strelka-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/strelka-logs.json index 089b5d4f8..89e9bbe8e 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/strelka-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/strelka-logs.json @@ -25,5 +25,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/suricata-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/suricata-logs.json index a9d857b24..c3b04fd86 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/suricata-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/suricata-logs.json @@ -25,5 +25,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/syslog-tcp-514.json b/salt/elasticfleet/files/integrations/grid-nodes_general/syslog-tcp-514.json index 80baa45ca..4088f5a87 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/syslog-tcp-514.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/syslog-tcp-514.json @@ -28,5 +28,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/syslog-udp-514.json b/salt/elasticfleet/files/integrations/grid-nodes_general/syslog-udp-514.json index 653c788b5..ad32a6964 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/syslog-udp-514.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/syslog-udp-514.json @@ -29,5 +29,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/system-grid-nodes.json b/salt/elasticfleet/files/integrations/grid-nodes_general/system-grid-nodes.json index a5c4c3e81..8e6bf7958 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/system-grid-nodes.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/system-grid-nodes.json @@ -36,5 +36,6 @@ "system-system/metrics": { "enabled": false } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_heavy/osquery-grid-nodes.json b/salt/elasticfleet/files/integrations/grid-nodes_heavy/osquery-grid-nodes.json index b1454d4bd..b1d69a44f 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_heavy/osquery-grid-nodes.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_heavy/osquery-grid-nodes.json @@ -16,5 +16,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_heavy/system-grid-nodes.json b/salt/elasticfleet/files/integrations/grid-nodes_heavy/system-grid-nodes.json index 3df514f0b..6c42086bc 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_heavy/system-grid-nodes.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_heavy/system-grid-nodes.json @@ -36,5 +36,6 @@ "system-system/metrics": { "enabled": false } - } + }, + "force": true } From 042e5ae9f0a3aee89a9aa6246bb5d172a52a2bcf Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Thu, 14 Dec 2023 12:46:28 -0500 Subject: [PATCH 20/21] https://github.com/Security-Onion-Solutions/securityonion/issues/12021 --- salt/soc/defaults.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 3655019e4..414bc3a53 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1348,7 +1348,7 @@ soc: showSubtitle: true - name: Firewall description: Firewall events grouped by action - query: 'tags:firewall | groupby rule.action' + query: 'observer.type:firewall | groupby event.action' showSubtitle: true dashboards: advanced: true @@ -1551,7 +1551,7 @@ soc: query: 'tags:s7* | groupby -sankey event.dataset source.ip destination.ip | groupby event.dataset | groupby source.ip | groupby destination.ip | groupby destination.port' - name: Firewall description: Firewall logs - query: 'tags:firewall | groupby -sankey rule.action interface.name | groupby rule.action | groupby interface.name | groupby network.transport | groupby source.ip | groupby destination.ip | groupby destination.port' + query: 'observer.type:firewall | groupby -sankey event.action observer.ingress.interface.name | groupby event.action | groupby observer.ingress.interface.name | groupby network.type | groupby network.transport | groupby source.ip | groupby destination.ip | groupby destination.port' - name: VLAN description: VLAN (Virtual Local Area Network) tagged logs query: '* AND _exists_:network.vlan.id | groupby network.vlan.id | groupby source.ip | groupby -sankey source.ip destination.ip | groupby destination.ip | groupby destination.port | groupby event.dataset | groupby event.module | groupby observer.name | groupby source.geo.country_name | groupby destination.geo.country_name' From 8779fb8cbc69657f460b7e0935779eb36c241f5d Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Thu, 14 Dec 2023 13:30:52 -0500 Subject: [PATCH 21/21] Update defaults.yaml --- salt/soc/defaults.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 414bc3a53..f440bd57b 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -470,6 +470,18 @@ soc: - rule.action - rule.reason - network.community_id + ':pfsense:': + - soc_timestamp + - source.ip + - source.port + - destination.ip + - destination.port + - network.transport + - network.direction + - observer.ingress.interface.name + - event.action + - event.reason + - network.community_id ':osquery:': - soc_timestamp - source.ip