Merge branch '2.4/dev' into fix/receiver

.github/workflows/contrib.yml

@@ -11,7 +11,7 @@ jobs:
     steps:
       - name: "Contributor Check"
         if: (github.event.comment.body == 'recheck' || github.event.comment.body == 'I have read the CLA Document and I hereby sign the CLA') || github.event_name == 'pull_request_target'
-        uses: cla-assistant/github-action@v2.1.3-beta
+        uses: cla-assistant/github-action@v2.3.1
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           PERSONAL_ACCESS_TOKEN : ${{ secrets.PERSONAL_ACCESS_TOKEN }}

pillar/top.sls

@@ -61,8 +61,6 @@ base:
     - elastalert.adv_elastalert
     - backup.soc_backup
     - backup.adv_backup
-    - curator.soc_curator
-    - curator.adv_curator
     - soctopus.soc_soctopus
     - soctopus.adv_soctopus
     - minions.{{ grains.id }}
@@ -113,8 +111,6 @@ base:
     - kibana.adv_kibana
     - strelka.soc_strelka
     - strelka.adv_strelka
-    - curator.soc_curator
-    - curator.adv_curator
     - kratos.soc_kratos
     - kratos.adv_kratos
     - redis.soc_redis
@@ -172,8 +168,6 @@ base:
     - kibana.adv_kibana
     - strelka.soc_strelka
     - strelka.adv_strelka
-    - curator.soc_curator
-    - curator.adv_curator
     - backup.soc_backup
     - backup.adv_backup
     - zeek.soc_zeek
@@ -194,8 +188,6 @@ base:
     - logstash.adv_logstash
     - elasticsearch.soc_elasticsearch
     - elasticsearch.adv_elasticsearch
-    - curator.soc_curator
-    - curator.adv_curator
     - redis.soc_redis
     - redis.adv_redis
     - zeek.soc_zeek
@@ -268,8 +260,6 @@ base:
     - soctopus.adv_soctopus
     - kibana.soc_kibana
     - kibana.adv_kibana
-    - curator.soc_curator
-    - curator.adv_curator
     - backup.soc_backup
     - backup.adv_backup
     - kratos.soc_kratos

salt/allowed_states.map.jinja

@@ -219,10 +219,6 @@
     {% do allowed_states.append('kibana.secrets') %}
   {% endif %}
 
-  {% if grains.role in ['so-eval', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-manager'] %}
-    {% do allowed_states.append('curator') %}
-  {% endif %}
-
   {% if grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-managersearch'] %}
     {% do allowed_states.append('elastalert') %}
   {% endif %}

salt/common/tools/sbin/so-image-common

@@ -42,7 +42,6 @@ container_list() {
     )
   elif [ $MANAGERCHECK != 'so-helix' ]; then
     TRUSTED_CONTAINERS=(
-      "so-curator"
       "so-elastalert"
       "so-elastic-agent"
       "so-elastic-agent-builder"

salt/common/tools/sbin/so-log-check

@@ -144,6 +144,7 @@ if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|status 200"                   # false positive (request successful, contained error string in content)
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|app_layer.error"              # false positive (suricata 7) in stats.log e.g. app_layer.error.imap.parser | Total | 0
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|is not an ip string literal"  # false positive (Open Canary logging out blank IP addresses)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|cannot join on an empty table" # false positive (InfluxDB flux query, import nodes)
 fi
 
 if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then

salt/common/tools/sbin/so-nsm-clear

@@ -41,8 +41,13 @@ done
 if [ $SKIP -ne 1 ]; then
         # Inform user we are about to delete all data
         echo
-        echo "This script will delete all NIDS data (PCAP, Suricata, Zeek)" 
+        echo "This script will delete all NSM data from /nsm."
-        echo "If you would like to proceed, please type "AGREE" and hit ENTER."
+        echo
+        echo "This includes Suricata data, Zeek data, and full packet capture (PCAP)."
+        echo
+        echo "This will NOT delete any Suricata or Zeek logs that have already been ingested into Elasticsearch."
+        echo
+        echo "If you would like to proceed, then type AGREE and press ENTER."
         echo
         # Read user input
         read INPUT
@@ -54,8 +59,8 @@ delete_pcap() {
   [ -d $PCAP_DATA ] && so-pcap-stop && rm -rf $PCAP_DATA/* && so-pcap-start
 }
 delete_suricata() {
-  SURI_LOG="/opt/so/log/suricata/eve.json"
+  SURI_LOG="/nsm/suricata/"
-  [ -f $SURI_LOG ] && so-suricata-stop && rm -f $SURI_LOG && so-suricata-start
+  [ -d $SURI_LOG ] && so-suricata-stop && rm -rf $SURI_LOG/* && so-suricata-start
 }
 delete_zeek() {
   ZEEK_LOG="/nsm/zeek/logs/"

salt/curator/config.sls

@@ -1,81 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-{% from 'allowed_states.map.jinja' import allowed_states %}
-{% if sls.split('.')[0] in allowed_states %}
-{%   from "curator/map.jinja" import CURATORMERGED %}
-
-# Create the group
-curatorgroup:
-  group.present:
-    - name: curator
-    - gid: 934
-
-# Add user
-curator:
-  user.present:
-    - uid: 934
-    - gid: 934
-    - home: /opt/so/conf/curator
-    - createhome: False
-
-# Create the log directory
-curlogdir:
-  file.directory:
-    - name: /opt/so/log/curator
-    - user: 934
-    - group: 939
-
-curactiondir:
-  file.directory:
-    - name: /opt/so/conf/curator/action
-    - user: 934
-    - group: 939
-    - makedirs: True
-
-actionconfs:
-  file.recurse:
-    - name: /opt/so/conf/curator/action
-    - source: salt://curator/files/action
-    - user: 934
-    - group: 939
-    - template: jinja
-    - defaults:
-        CURATORMERGED: {{ CURATORMERGED.elasticsearch.index_settings }}
-        
-curconf:
-  file.managed:
-    - name: /opt/so/conf/curator/curator.yml
-    - source: salt://curator/files/curator.yml
-    - user: 934
-    - group: 939
-    - mode: 660
-    - template: jinja
-    - show_changes: False
-
-curator_sbin:
-  file.recurse:
-    - name: /usr/sbin
-    - source: salt://curator/tools/sbin
-    - user: 934
-    - group: 939
-    - file_mode: 755
-
-curator_sbin_jinja:
-  file.recurse:
-    - name: /usr/sbin
-    - source: salt://curator/tools/sbin_jinja
-    - user: 934
-    - group: 939 
-    - file_mode: 755
-    - template: jinja
-
-{% else %}
-
-{{sls}}_state_not_allowed:
-  test.fail_without_changes:
-    - name: {{sls}}_state_not_allowed
-
-{% endif %}

salt/curator/defaults.yaml

@@ -1,34 +0,0 @@
-curator:
-  enabled: False
-  elasticsearch:
-    index_settings:
-      so-beats:
-        delete: 365
-      so-elasticsearch:
-        delete: 365
-      so-firewall:
-        delete: 365
-      so-ids:
-        delete: 365
-      so-import:
-        delete: 73001
-      so-kratos:
-        delete: 365
-      so-kibana:
-        delete: 365
-      so-logstash:
-        delete: 365
-      so-netflow:
-        delete: 365
-      so-osquery:
-        delete: 365
-      so-ossec:
-        delete: 365
-      so-redis:
-        delete: 365
-      so-strelka:
-        delete: 365
-      so-syslog:
-        delete: 365
-      so-zeek:
-        delete: 365

salt/curator/disabled.sls

@@ -1,35 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-{% from 'allowed_states.map.jinja' import allowed_states %}
-{% if sls.split('.')[0] in allowed_states %}
-
-include:
-  - curator.sostatus
-  
-so-curator:
-  docker_container.absent:
-    - force: True
-
-so-curator_so-status.disabled:
-  file.comment:
-    - name: /opt/so/conf/so-status/so-status.conf
-    - regex: ^so-curator$
-
-so-curator-cluster-close:
-  cron.absent:
-    - identifier: so-curator-cluster-close
-
-so-curator-cluster-delete:
-  cron.absent:
-    - identifier: so-curator-cluster-delete
-
-{% else %}
-
-{{sls}}_state_not_allowed:
-  test.fail_without_changes:
-    - name: {{sls}}_state_not_allowed
-
-{% endif %}

salt/curator/enabled.sls

@@ -1,81 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-{% from 'allowed_states.map.jinja' import allowed_states %}
-{% if sls.split('.')[0] in allowed_states %}
-{%   from 'vars/globals.map.jinja' import GLOBALS %}
-{%   from 'docker/docker.map.jinja' import DOCKER %}
-
-include:
-  - curator.config
-  - curator.sostatus
-
-so-curator:
-  docker_container.running:
-    - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-curator:{{ GLOBALS.so_version }}
-    - start: True
-    - hostname: curator
-    - name: so-curator
-    - user: curator
-    - networks:
-      - sobridge:
-        - ipv4_address: {{ DOCKER.containers['so-curator'].ip }}
-    - interactive: True
-    - tty: True
-    - binds:
-      - /opt/so/conf/curator/curator.yml:/etc/curator/config/curator.yml:ro
-      - /opt/so/conf/curator/action/:/etc/curator/action:ro
-      - /opt/so/log/curator:/var/log/curator:rw
-      {% if DOCKER.containers['so-curator'].custom_bind_mounts %}
-        {% for BIND in DOCKER.containers['so-curator'].custom_bind_mounts %}
-      - {{ BIND }}
-        {% endfor %}
-      {% endif %}
-      {% if DOCKER.containers['so-curator'].extra_hosts %}
-    - extra_hosts:
-        {% for XTRAHOST in DOCKER.containers['so-curator'].extra_hosts %}
-      - {{ XTRAHOST }}
-        {% endfor %}
-      {% endif %}
-      {% if DOCKER.containers['so-curator'].extra_env %}
-    - environment:
-        {% for XTRAENV in DOCKER.containers['so-curator'].extra_env %}
-      - {{ XTRAENV }}
-        {% endfor %}
-      {% endif %}
-    - require:
-      - file: actionconfs
-      - file: curconf
-      - file: curlogdir
-    - watch:
-      - file: curconf
-
-delete_so-curator_so-status.disabled:
-  file.uncomment:
-    - name: /opt/so/conf/so-status/so-status.conf
-    - regex: ^so-curator$
-
-so-curator-cluster-close:
-  cron.absent:
-    - identifier: so-curator-cluster-close
-
-so-curator-cluster-delete:
-  cron.present:
-    - name: /usr/sbin/so-curator-cluster-delete > /opt/so/log/curator/cron-cluster-delete.log 2>&1
-    - identifier: so-curator-cluster-delete
-    - user: root
-    - minute: '*/5'
-    - hour: '*'
-    - daymonth: '*'
-    - month: '*'
-    - dayweek: '*'
-
-{% else %}
-
-{{sls}}_state_not_allowed:
-  test.fail_without_changes:
-    - name: {{sls}}_state_not_allowed
-
-{% endif %}

salt/curator/files/action/delete.yml

@@ -1,32 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-{% import_yaml 'elasticsearch/defaults.yaml' as ELASTICDEFAULTS %}
-{% set ELASTICMERGED = salt['pillar.get']('elasticsearch:retention', ELASTICDEFAULTS.elasticsearch.retention, merge=true) %}
-
-{{ ELASTICMERGED.retention_pct }}
-
-{%- set log_size_limit = salt['pillar.get']('elasticsearch:log_size_limit') %}
-actions:
-  1:
-    action: delete_indices
-    description: >-
-      Delete indices when {{log_size_limit}}(GB) is exceeded.
-    options:
-      allow_ilm_indices: True
-      ignore_empty_list: True
-      disable_action: False
-    filters:
-    - filtertype: pattern
-      kind: regex
-      value: '^(logstash-.*|so-.*|.ds-logs-.*-so.*)$'
-    - filtertype: pattern
-      kind: regex
-      value: '^(so-case.*)$'
-      exclude: True
-    - filtertype: space
-      source: creation_date
-      use_age: True
-      disk_space: {{log_size_limit}}

salt/curator/files/action/so-beats-delete.yml

@@ -1,27 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-{%- set DELETE_DAYS = CURATORMERGED['so-beats'].delete %}
-actions:
-  1:
-    action: delete_indices
-    description: >-
-      Delete beats indices when older than {{ DELETE_DAYS }} days.
-    options:
-      ignore_empty_list: True
-      disable_action: False
-    filters:
-    - filtertype: pattern
-      kind: regex
-      value: '^(logstash-beats.*|so-beats.*)$'
-    - filtertype: age
-      source: name
-      direction: older
-      timestring: '%Y.%m.%d'
-      unit: days
-      unit_count: {{ DELETE_DAYS }}
-      exclude:
-  
-      

salt/curator/files/action/so-elasticsearch-delete.yml

@@ -1,27 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-{%- set DELETE_DAYS = CURATORMERGED['so-elasticsearch'].delete %}
-actions:
-  1:
-    action: delete_indices
-    description: >-
-      Delete elasticsearch indices when older than {{ DELETE_DAYS }} days.
-    options:
-      ignore_empty_list: True
-      disable_action: False
-    filters:
-    - filtertype: pattern
-      kind: regex
-      value: '^(logstash-elasticsearch.*|so-elasticsearch.*)$'
-    - filtertype: age
-      source: name
-      direction: older
-      timestring: '%Y.%m.%d'
-      unit: days
-      unit_count: {{ DELETE_DAYS }}
-      exclude:
-  
-      

salt/curator/files/action/so-firewall-delete.yml

@@ -1,28 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-
-{%- set DELETE_DAYS = CURATORMERGED['so-firewall'].delete %}
-actions:
-  1:
-    action: delete_indices
-    description: >-
-      Delete firewall indices when older than {{ DELETE_DAYS }} days.
-    options:
-      ignore_empty_list: True
-      disable_action: False
-    filters:
-    - filtertype: pattern
-      kind: regex
-      value: '^(logstash-firewall.*|so-firewall.*)$'
-    - filtertype: age
-      source: name
-      direction: older
-      timestring: '%Y.%m.%d'
-      unit: days
-      unit_count: {{ DELETE_DAYS }}
-      exclude:
-  
-      

salt/curator/files/action/so-ids-delete.yml

@@ -1,28 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-
-{%- set DELETE_DAYS = CURATORMERGED['so-ids'].delete %}
-actions:
-  1:
-    action: delete_indices
-    description: >-
-      Delete IDS indices when older than {{ DELETE_DAYS }} days.
-    options:
-      ignore_empty_list: True
-      disable_action: False
-    filters:
-    - filtertype: pattern
-      kind: regex
-      value: '^(logstash-ids.*|so-ids.*)$'
-    - filtertype: age
-      source: name
-      direction: older
-      timestring: '%Y.%m.%d'
-      unit: days
-      unit_count: {{ DELETE_DAYS }}
-      exclude:
-  
-      

salt/curator/files/action/so-import-delete.yml

@@ -1,27 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-{%- set DELETE_DAYS = CURATORMERGED['so-import'].delete %}
-actions:
-  1:
-    action: delete_indices
-    description: >-
-      Delete import indices when older than {{ DELETE_DAYS }} days.
-    options:
-      ignore_empty_list: True
-      disable_action: False
-    filters:
-    - filtertype: pattern
-      kind: regex
-      value: '^(logstash-import.*|so-import.*)$'
-    - filtertype: age
-      source: name
-      direction: older
-      timestring: '%Y.%m.%d'
-      unit: days
-      unit_count: {{ DELETE_DAYS }}
-      exclude:
-  
-      

salt/curator/files/action/so-kibana-delete.yml

@@ -1,27 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-{%- set DELETE_DAYS = CURATORMERGED['so-kibana'].delete %}
-actions:
-  1:
-    action: delete_indices
-    description: >-
-      Delete kibana indices when older than {{ DELETE_DAYS }} days.
-    options:
-      ignore_empty_list: True
-      disable_action: False
-    filters:
-    - filtertype: pattern
-      kind: regex
-      value: '^(logstash-kibana.*|so-kibana.*)$'
-    - filtertype: age
-      source: name
-      direction: older
-      timestring: '%Y.%m.%d'
-      unit: days
-      unit_count: {{ DELETE_DAYS }}
-      exclude:
-  
-      

salt/curator/files/action/so-kratos-delete.yml

@@ -1,27 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-{%- set DELETE_DAYS = CURATORMERGED['so-kratos'].delete %}
-actions:
-  1:
-    action: delete_indices
-    description: >-
-      Delete kratos indices when older than {{ DELETE_DAYS }} days.
-    options:
-      ignore_empty_list: True
-      disable_action: False
-    filters:
-    - filtertype: pattern
-      kind: regex
-      value: '^(logstash-kratos.*|so-kratos.*)$'
-    - filtertype: age
-      source: name
-      direction: older
-      timestring: '%Y.%m.%d'
-      unit: days
-      unit_count: {{ DELETE_DAYS }}
-      exclude:
-  
-      

salt/curator/files/action/so-logstash-delete.yml

@@ -1,27 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-{%- set DELETE_DAYS = CURATORMERGED['so-logstash'].delete %}
-actions:
-  1:
-    action: delete_indices
-    description: >-
-      Delete logstash indices when older than {{ DELETE_DAYS }} days.
-    options:
-      ignore_empty_list: True
-      disable_action: False
-    filters:
-    - filtertype: pattern
-      kind: regex
-      value: '^(logstash-logstash.*|so-logstash.*)$'
-    - filtertype: age
-      source: name
-      direction: older
-      timestring: '%Y.%m.%d'
-      unit: days
-      unit_count: {{ DELETE_DAYS }}
-      exclude:
-  
-      

salt/curator/files/action/so-netflow-delete.yml

@@ -1,27 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-{%- set DELETE_DAYS = CURATORMERGED['so-netflow'].delete %}
-actions:
-  1:
-    action: delete_indices
-    description: >-
-      Delete netflow indices when older than {{ DELETE_DAYS }} days.
-    options:
-      ignore_empty_list: True
-      disable_action: False
-    filters:
-    - filtertype: pattern
-      kind: regex
-      value: '^(logstash-netflow.*|so-netflow.*)$'
-    - filtertype: age
-      source: name
-      direction: older
-      timestring: '%Y.%m.%d'
-      unit: days
-      unit_count: {{ DELETE_DAYS }}
-      exclude:
-  
-      

salt/curator/files/action/so-osquery-delete.yml

@@ -1,27 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-{%- set DELETE_DAYS = CURATORMERGED['so-osquery'].delete %}
-actions:
-  1:
-    action: delete_indices
-    description: >-
-      Delete import indices when older than {{ DELETE_DAYS }} days.
-    options:
-      ignore_empty_list: True
-      disable_action: False
-    filters:
-    - filtertype: pattern
-      kind: regex
-      value: '^(logstash-osquery.*|so-osquery.*)$'
-    - filtertype: age
-      source: name
-      direction: older
-      timestring: '%Y.%m.%d'
-      unit: days
-      unit_count: {{ DELETE_DAYS }}
-      exclude:
-  
-      

salt/curator/files/action/so-ossec-delete.yml

@@ -1,27 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-{%- set DELETE_DAYS = CURATORMERGED['so-ossec'].delete %}
-actions:
-  1:
-    action: delete_indices
-    description: >-
-      Delete ossec indices when older than {{ DELETE_DAYS }} days.
-    options:
-      ignore_empty_list: True
-      disable_action: False
-    filters:
-    - filtertype: pattern
-      kind: regex
-      value: '^(logstash-ossec.*|so-ossec.*)$'
-    - filtertype: age
-      source: name
-      direction: older
-      timestring: '%Y.%m.%d'
-      unit: days
-      unit_count: {{ DELETE_DAYS }}
-      exclude:
-  
-      

salt/curator/files/action/so-redis-delete.yml

@@ -1,27 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-{%- set DELETE_DAYS = CURATORMERGED['so-redis'].delete %}
-actions:
-  1:
-    action: delete_indices
-    description: >-
-      Delete redis indices when older than {{ DELETE_DAYS }} days.
-    options:
-      ignore_empty_list: True
-      disable_action: False
-    filters:
-    - filtertype: pattern
-      kind: regex
-      value: '^(logstash-redis.*|so-redis.*)$'
-    - filtertype: age
-      source: name
-      direction: older
-      timestring: '%Y.%m.%d'
-      unit: days
-      unit_count: {{ DELETE_DAYS }}
-      exclude:
-  
-      

salt/curator/files/action/so-strelka-delete.yml

@@ -1,27 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-{%- set DELETE_DAYS = CURATORMERGED['so-strelka'].delete %}
-actions:
-  1:
-    action: delete_indices
-    description: >-
-      Delete Strelka indices when older than {{ DELETE_DAYS }} days.
-    options:
-      ignore_empty_list: True
-      disable_action: False
-    filters:
-    - filtertype: pattern
-      kind: regex
-      value: '^(logstash-strelka.*|so-strelka.*)$'
-    - filtertype: age
-      source: name
-      direction: older
-      timestring: '%Y.%m.%d'
-      unit: days
-      unit_count: {{ DELETE_DAYS }}
-      exclude:
-  
-      

salt/curator/files/action/so-syslog-delete.yml

@@ -1,27 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-{%- set DELETE_DAYS = CURATORMERGED['so-syslog'].delete %}
-actions:
-  1:
-    action: delete_indices
-    description: >-
-      Delete syslog indices when older than {{ DELETE_DAYS }} days.
-    options:
-      ignore_empty_list: True
-      disable_action: False
-    filters:
-    - filtertype: pattern
-      kind: regex
-      value: '^(logstash-syslog.*|so-syslog.*)$'
-    - filtertype: age
-      source: name
-      direction: older
-      timestring: '%Y.%m.%d'
-      unit: days
-      unit_count: {{ DELETE_DAYS }}
-      exclude:
-  
-      

salt/curator/files/action/so-zeek-delete.yml

@@ -1,27 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-{%- set DELETE_DAYS = CURATORMERGED['so-zeek'].delete %}
-actions:
-  1:
-    action: delete_indices
-    description: >-
-      Delete Zeek indices when older than {{ DELETE_DAYS }} days.
-    options:
-      ignore_empty_list: True
-      disable_action: False
-    filters:
-    - filtertype: pattern
-      kind: regex
-      value: '^(logstash-zeek.*|so-zeek.*)$'
-    - filtertype: age
-      source: name
-      direction: older
-      timestring: '%Y.%m.%d'
-      unit: days
-      unit_count: {{ DELETE_DAYS }}
-      exclude:
-  
-      

salt/curator/files/curator.yml

@@ -1,40 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-{% from 'vars/globals.map.jinja' import GLOBALS %}
-{% if GLOBALS.role in ['so-searchnode', 'so-heavynode'] %}
-  {%- set elasticsearch = GLOBALS.node_ip -%}
-{% elif GLOBALS.role in ['so-eval', 'so-managersearch', 'so-standalone', 'so-manager'] %}
-  {%- set elasticsearch = GLOBALS.manager_ip -%}
-{%- endif %}
-{%-   set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %}
-{%-   set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %}
-
----
-# Remember, leave a key empty if there is no value.  None will be a string,
-# not a Python "NoneType"
-elasticsearch:
-  client:
-    hosts:
-      - https://{{elasticsearch}}:9200
-    cloud_id:
-    ca_certs:
-    client_cert:
-    client_key:
-    verify_certs: False
-    request_timeout: 30
-  other_settings:
-    api_key:
-      id:
-      api_key:
-    master_only: False
-    username: "{{ ES_USER }}"
-    password: "{{ ES_PASS }}"
-
-logging:
-  loglevel: INFO
-  logfile: '/var/log/curator/curator.log'
-  logformat: default
-  blacklist: ['elasticsearch', 'urllib3']

salt/curator/init.sls

@@ -1,13 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-{% from 'curator/map.jinja' import CURATORMERGED %}
-
-include:
-{% if CURATORMERGED.enabled %}
-  - curator.enabled
-{% else %}
-  - curator.disabled
-{% endif %}

salt/curator/map.jinja

@@ -1,7 +0,0 @@
-{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-   or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-   https://securityonion.net/license; you may not use this file except in compliance with the
-   Elastic License 2.0. #}
-
-{% import_yaml 'curator/defaults.yaml' as CURATORDEFAULTS %}
-{% set CURATORMERGED = salt['pillar.get']('curator', CURATORDEFAULTS.curator, merge=true) %}

salt/curator/soc_curator.yaml

@@ -1,39 +0,0 @@
-curator:
-  enabled:
-    description: You can enable or disable Curator.
-    helpLink: curator.html
-  elasticsearch:
-    index_settings:
-      so-beats:
-        delete: &delete
-          description: Age, in days, when Curator deletes the index.
-          helpLink: curator.html
-          forcedType: int
-      so-elasticsearch:
-        delete: *delete
-      so-firewall:
-        delete: *delete
-      so-ids:
-        delete: *delete
-      so-import:
-        delete: *delete
-      so-kratos:
-        delete: *delete
-      so-kibana:
-        delete: *delete
-      so-logstash:
-        delete: *delete
-      so-netflow:
-        delete: *delete
-      so-osquery:
-        delete: *delete
-      so-ossec:
-        delete: *delete
-      so-redis:
-        delete: *delete
-      so-strelka:
-        delete: *delete
-      so-syslog:
-        delete: *delete
-      so-zeek:
-        delete: *delete

salt/curator/sostatus.sls

@@ -1,21 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-{% from 'allowed_states.map.jinja' import allowed_states %}
-{% if sls.split('.')[0] in allowed_states %}
-
-append_so-curator_so-status.conf:
-  file.append:
-    - name: /opt/so/conf/so-status/so-status.conf
-    - text: so-curator
-    - unless: grep -q so-curator /opt/so/conf/so-status/so-status.conf
-
-{% else %}
-
-{{sls}}_state_not_allowed:
-  test.fail_without_changes:
-    - name: {{sls}}_state_not_allowed
-
-{% endif %}

salt/curator/tools/sbin/so-curator-delete

@@ -1,17 +0,0 @@
-#!/bin/bash
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-
-APP=delete
-lf=/tmp/$APP-pidLockFile
-# create empty lock file if none exists
-cat /dev/null >> $lf
-read lastPID < $lf
-# if lastPID is not null and a process with that pid exists , exit
-[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit
-echo $$ > $lf
-
-docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/delete.yml > /dev/null 2>&1

salt/curator/tools/sbin/so-curator-restart

@@ -1,12 +0,0 @@
-#!/bin/bash
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-
-
-. /usr/sbin/so-common
-
-/usr/sbin/so-restart curator $1

salt/curator/tools/sbin/so-curator-start

@@ -1,12 +0,0 @@
-#!/bin/bash
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-
-
-. /usr/sbin/so-common
-
-/usr/sbin/so-start curator $1

salt/curator/tools/sbin/so-curator-stop

@@ -1,12 +0,0 @@
-#!/bin/bash
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-
-
-. /usr/sbin/so-common
-
-/usr/sbin/so-stop curator $1

salt/docker/defaults.yaml

@@ -159,11 +159,6 @@ docker:
       custom_bind_mounts: []
       extra_hosts: []
       extra_env: []
-    'so-curator':
-      final_octet: 43
-      custom_bind_mounts: []
-      extra_hosts: []
-      extra_env: []
     'so-elastic-fleet-package-registry':
       final_octet: 44
       port_bindings:

salt/docker/soc_docker.yaml

@@ -60,7 +60,6 @@ docker:
     so-strelka-gatekeeper: *dockerOptions
     so-strelka-coordinator: *dockerOptions
     so-elastalert: *dockerOptions
-    so-curator: *dockerOptions
     so-elastic-fleet-package-registry: *dockerOptions
     so-idh: *dockerOptions
     so-elastic-agent: *dockerOptions

salt/elasticfleet/defaults.yaml

@@ -76,6 +76,7 @@ elasticfleet:
     - pulse_connect_secure
     - redis
     - sentinel_one
+    - snort
     - snyk
     - sonicwall_firewall
     - sophos
@@ -85,9 +86,12 @@ elasticfleet:
     - tcp
     - tenable_sc
     - ti_abusech
+    - ti_anomali
+    - ti_cybersixgill
     - ti_misp
     - ti_otx
     - ti_recordedfuture
+    - ti_threatq
     - udp
     - vsphere
     - windows

salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/import-zeek-logs.json

@@ -27,6 +27,7 @@
         }
       }
     }
-  }
+  },
+  "force": true
 }
 {%- endraw -%}

salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/zeek-logs.json

@@ -28,6 +28,7 @@
         }
       }
     }
-  }
+  },
+  "force": true
 }
 {%- endraw -%}

salt/elasticfleet/files/integrations-optional/sublime_platform.json

@@ -39,6 +39,7 @@
         }
       }
     }
-  }
+  },
+  "force": true
 }
 {%- endraw -%}

salt/elasticfleet/files/integrations/endpoints-initial/osquery.json

@@ -16,5 +16,6 @@
         }
       }
     }
-  }
+  },
+  "force": true
 }

salt/elasticfleet/files/integrations/endpoints-initial/system-endpoints.json

@@ -73,5 +73,6 @@
     "system-system/metrics": {
       "enabled": false
     }
-  }
+  },
+  "force": true
 }

salt/elasticfleet/files/integrations/endpoints-initial/windows-endpoints.json

@@ -63,5 +63,6 @@
         }
       }
     }
-  }
+  },
+  "force": true
 }

salt/elasticfleet/files/integrations/grid-nodes_general/elasticsearch-logs.json

@@ -102,5 +102,6 @@
         }
       }
     }
-  }
+  },
+  "force": true
 }

salt/elasticfleet/files/integrations/grid-nodes_general/idh-logs.json

@@ -25,5 +25,6 @@
         }
       }
     }
-  }
+  },
+  "force": true
 }

salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json

@@ -28,5 +28,6 @@
         }
       }
     }
-  }
+  },
+  "force": true
 }

salt/elasticfleet/files/integrations/grid-nodes_general/import-suricata-logs.json

@@ -25,5 +25,6 @@
         }
       }
     }
-  }
+  },
+  "force": true
 }

salt/elasticfleet/files/integrations/grid-nodes_general/kratos-logs.json

@@ -25,5 +25,6 @@
         }
       }
     }
-  }
+  },
+  "force": true
 }

salt/elasticfleet/files/integrations/grid-nodes_general/osquery-grid-nodes.json

@@ -16,5 +16,6 @@
         }
       }
     }
-  }
+  },
+  "force": true
 }

salt/elasticfleet/files/integrations/grid-nodes_general/redis-logs.json

@@ -72,5 +72,6 @@
         }
       }
     }
-  }
+  },
+  "force": true
 }

salt/elasticfleet/files/integrations/grid-nodes_general/soc-auth-sync-logs.json

@@ -25,5 +25,6 @@
         }
       }
     }
-  }
+  },
+  "force": true
 }

salt/elasticfleet/files/integrations/grid-nodes_general/soc-salt-relay-logs.json

@@ -25,5 +25,6 @@
         }
       }
     }
-  }
+  },
+  "force": true
 }

salt/elasticfleet/files/integrations/grid-nodes_general/soc-sensoroni-logs.json

@@ -25,5 +25,6 @@
         }
       }
     }
-  }
+  },
+  "force": true
 }

salt/elasticfleet/files/integrations/grid-nodes_general/soc-server-logs.json

@@ -25,5 +25,6 @@
         }
       }
     }
-  }
+  },
+  "force": true
 }

salt/elasticfleet/files/integrations/grid-nodes_general/strelka-logs.json

@@ -25,5 +25,6 @@
         }
       }
     }
-  }
+  },
+  "force": true
 }

salt/elasticfleet/files/integrations/grid-nodes_general/suricata-logs.json

@@ -25,5 +25,6 @@
         }
       }
     }
-  }
+  },
+  "force": true
 }

salt/elasticfleet/files/integrations/grid-nodes_general/syslog-tcp-514.json

@@ -28,5 +28,6 @@
         }
       }
     }
-  }
+  },
+  "force": true
 }

salt/elasticfleet/files/integrations/grid-nodes_general/syslog-udp-514.json

@@ -29,5 +29,6 @@
         }
       }
     }
-  }
+  },
+  "force": true
 }

salt/elasticfleet/files/integrations/grid-nodes_general/system-grid-nodes.json

@@ -36,5 +36,6 @@
     "system-system/metrics": {
       "enabled": false
     }
-  }
+  },
+  "force": true
 }

salt/elasticfleet/files/integrations/grid-nodes_heavy/osquery-grid-nodes.json

@@ -16,5 +16,6 @@
         }
       }
     }
-  }
+  },
+  "force": true
 }

salt/elasticfleet/files/integrations/grid-nodes_heavy/system-grid-nodes.json

@@ -36,5 +36,6 @@
     "system-system/metrics": {
       "enabled": false
     }
-  }
+  },
+  "force": true
 }

salt/elasticsearch/defaults.yaml

@@ -6737,6 +6737,50 @@ elasticsearch:
               set_priority:
                 priority: 50
             min_age: 30d
+    so-logs-snort_x_log:
+      index_sorting: False
+      index_template:
+        index_patterns:
+          - "logs-snort.log-*"
+        template:
+          settings:
+            index:
+              lifecycle:
+                name: so-logs-snort.log-logs
+              number_of_replicas: 0
+        composed_of:
+          - "logs-snort.log@package"
+          - "logs-snort.log@custom"
+          - "so-fleet_globals-1"
+          - "so-fleet_agent_id_verification-1"
+        priority: 501
+        data_stream:
+          hidden: false
+          allow_custom_routing: false
+      policy:
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 30d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
     so-logs-snyk_x_audit:
       index_sorting: false
       index_template:
@@ -7622,6 +7666,94 @@ elasticsearch:
               set_priority:
                 priority: 50
             min_age: 30d
+    so-logs-ti_anomali_x_threatstream:
+      index_sorting: False
+      index_template:
+        index_patterns:
+          - "logs-ti_anomali.threatstream-*"
+        template:
+          settings:
+            index:
+              lifecycle:
+                name: so-logs-ti_anomali.threatstream-logs
+              number_of_replicas: 0
+        composed_of:
+          - "logs-ti_anomali.threatstream@package"
+          - "logs-ti_anomali.threatstream@custom"
+          - "so-fleet_globals-1"
+          - "so-fleet_agent_id_verification-1"
+        priority: 501
+        data_stream:
+          hidden: false
+          allow_custom_routing: false
+      policy:
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 30d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
+    so-logs-ti_cybersixgill_x_threat:
+      index_sorting: False
+      index_template:
+        index_patterns:
+          - "logs-ti_cybersixgill.threat-*"
+        template:
+          settings:
+            index:
+              lifecycle:
+                name: so-logs-ti_cybersixgill.threat-logs
+              number_of_replicas: 0
+        composed_of:
+          - "logs-ti_cybersixgill.threat@package"
+          - "logs-ti_cybersixgill.threat@custom"
+          - "so-fleet_globals-1"
+          - "so-fleet_agent_id_verification-1"
+        priority: 501
+        data_stream:
+          hidden: false
+          allow_custom_routing: false
+      policy:
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 30d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
     so-logs-ti_misp_x_threat:
       index_sorting: false
       index_template:
@@ -7842,6 +7974,50 @@ elasticsearch:
               set_priority:
                 priority: 50
             min_age: 30d
+    so-logs-ti_threatq_x_threat:
+      index_sorting: False
+      index_template:
+        index_patterns:
+          - "logs-ti_threatq.threat-*"
+        template:
+          settings:
+            index:
+              lifecycle:
+                name: so-logs-ti_threatq.threat-logs
+              number_of_replicas: 0
+        composed_of:
+          - "logs-ti_threatq.threat@package"
+          - "logs-ti_threatq.threat@custom"
+          - "so-fleet_globals-1"
+          - "so-fleet_agent_id_verification-1"
+        priority: 501
+        data_stream:
+          hidden: false
+          allow_custom_routing: false
+      policy:
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 30d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
     so-logs-vsphere_x_log:
       index_sorting: False
       index_template:

salt/elasticsearch/disabled.sls

@@ -18,6 +18,10 @@ so-elasticsearch_so-status.disabled:
     - name: /opt/so/conf/so-status/so-status.conf
     - regex: ^so-elasticsearch$
 
+so-elasticsearch-indices-delete:
+  cron.absent:
+    - identifier: so-elasticsearch-indices-delete 
+
 {% else %}
 
 {{sls}}_state_not_allowed:

salt/elasticsearch/enabled.sls

@@ -26,7 +26,12 @@ so-elasticsearch:
     - networks:
       - sobridge:
         - ipv4_address: {{ DOCKER.containers['so-elasticsearch'].ip }}
-    - extra_hosts:  {{ LOGSTASH_NODES }}
+    - extra_hosts:
+    {% for node in LOGSTASH_NODES %}
+    {%   for hostname, ip in node.items() %}
+      - {{hostname}}:{{ip}}
+    {%   endfor %}
+    {% endfor %}
     {% if DOCKER.containers['so-elasticsearch'].extra_hosts %}
       {% for XTRAHOST in DOCKER.containers['so-elasticsearch'].extra_hosts %}
       - {{ XTRAHOST }}
@@ -195,6 +200,26 @@ so-elasticsearch-roles-load:
     - require:
       - docker_container: so-elasticsearch
       - file: elasticsearch_sbin_jinja
+{% if grains.role in ['so-eval', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-manager'] %}
+so-curator-cluster-close:
+  cron.absent:
+    - identifier: so-curator-cluster-close
+
+so-curator-cluster-delete:
+  cron.absent:
+    - identifier: so-curator-cluster-delete
+
+so-elasticsearch-indices-delete:
+  cron.present:
+    - name: /usr/sbin/so-elasticsearch-indices-delete > /opt/so/log/elasticsearch/cron-elasticsearch-indices-delete.log 2>&1
+    - identifier: so-elasticsearch-indices-delete
+    - user: root
+    - minute: '*/5'
+    - hour: '*'
+    - daymonth: '*'
+    - month: '*'
+    - dayweek: '*'
+{% endif %}
 {%   endif %}
 
 {% else %}

salt/elasticsearch/soc_elasticsearch.yaml

@@ -407,16 +407,20 @@ elasticsearch:
     so-logs-sentinel_one_x_group: *indexSettings
     so-logs-sentinel_one_x_threat: *indexSettings
     so-logs-sonicwall_firewall_x_log: *indexSettings
+    so-logs-snort_x_log: *indexSettings
     so-logs-symantec_endpoint_x_log: *indexSettings
     so-logs-ti_abusech_x_malware: *indexSettings
     so-logs-ti_abusech_x_malwarebazaar: *indexSettings
     so-logs-ti_abusech_x_threatfox: *indexSettings
     so-logs-ti_abusech_x_url: *indexSettings
+    so-logs-ti_anomali_x_threatstream: *indexSettings
+    so-logs-ti_cybersixgill_x_threat: *indexSettings
     so-logs-ti_misp_x_threat: *indexSettings
     so-logs-ti_misp_x_threat_attributes: *indexSettings
     so-logs-ti_otx_x_threat: *indexSettings
     so-logs-ti_recordedfuture_x_latest_ioc-template: *indexSettings
     so-logs-ti_recordedfuture_x_threat: *indexSettings
+    so-logs-ti_threatq_x_threat: *indexSettings
     so-logs-zscaler_zia_x_alerts: *indexSettings
     so-logs-zscaler_zia_x_dns: *indexSettings
     so-logs-zscaler_zia_x_firewall: *indexSettings

salt/elasticsearch/tools/sbin/so-elastic-clear

@@ -67,7 +67,7 @@ if [ $SKIP -ne 1 ]; then
         echo
         echo "This script will delete all data (documents, indices, etc.) in the Elasticsearch database."
         echo
-        echo "If you would like to proceed, please type "AGREE" and hit ENTER."
+        echo "If you would like to proceed, then type AGREE and press ENTER."
         echo
         # Read user input
         read INPUT

salt/elasticsearch/tools/sbin/so-elasticsearch-indices-delete

@@ -14,4 +14,4 @@ read lastPID < $lf
 [ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit
 echo $$ > $lf
 
-/usr/sbin/so-curator-cluster-delete-delete
+/usr/sbin/so-elasticsearch-indices-delete-delete

salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-indices-delete-delete

@@ -9,8 +9,8 @@
 {%- set ELASTICSEARCH_HOST = GLOBALS.node_ip -%}
 {%- set RETENTION = salt['pillar.get']('elasticsearch:retention', ELASTICDEFAULTS.elasticsearch.retention, merge=true) -%}
 
-LOG="/opt/so/log/curator/so-curator-cluster-delete.log"
+LOG="/opt/so/log/elasticsearch/so-elasticsearch-indices-delete.log"
-ALERT_LOG="/opt/so/log/curator/alert.log"
+ALERT_LOG="/opt/so/log/elasticsearch/indices-delete-alert.log"
 LOG_SIZE_LIMIT_GB=$(/usr/sbin/so-elasticsearch-cluster-space-total {{ RETENTION.retention_pct}})
 LOG_SIZE_LIMIT=$(( "$LOG_SIZE_LIMIT_GB" * 1000 * 1000 * 1000 ))
 ITERATION=0
@@ -51,8 +51,11 @@ while overlimit && [[ $ITERATION -lt $MAX_ITERATIONS ]]; do
       if [ "${INDEX}" != "${CURRENT_WRITE_INDEX}" ]; then
         # This should not be a write index, so we should be allowed to delete it
         printf "\n$(date) - Used disk space exceeds LOG_SIZE_LIMIT (${LOG_SIZE_LIMIT_GB} GB) - Deleting ${INDEX} index...\n" >> ${LOG}
-        /usr/sbin/so-elasticsearch-query ${INDEX} -XDELETE >> ${LOG} 2>&1
+	/usr/sbin/so-elasticsearch-query ${INDEX} -XDELETE >> ${LOG} 2>&1
       fi
+    else
+	    printf "\n$(date) - Used disk space exceeds LOG_SIZE_LIMIT (${LOG_SIZE_LIMIT_GB} GB) - There is only one backing index (${INDEX}).  Deleting ${DATASTREAM} data stream...\n" >> ${LOG}
+      /usr/sbin/so-elasticsearch-query _data_stream/$DATASTREAM -XDELETE >> ${LOG} 2>&1
     fi
     if ! overlimit ; then
       exit

salt/firewall/containers.map.jinja

@@ -2,7 +2,6 @@
 
 {% if GLOBALS.role == 'so-eval' %}
 {% set NODE_CONTAINERS = [
-     'so-curator',
      'so-dockerregistry',
      'so-elasticsearch',
      'so-elastic-fleet',
@@ -25,7 +24,6 @@
 
 {% elif GLOBALS.role == 'so-manager' or GLOBALS.role == 'so-standalone' or GLOBALS.role == 'so-managersearch' %}
 {% set NODE_CONTAINERS = [
-     'so-curator',
      'so-dockerregistry',
      'so-elasticsearch',
      'so-elastic-fleet',
@@ -56,7 +54,6 @@
 
 {% elif GLOBALS.role == 'so-heavynode' %}
 {% set NODE_CONTAINERS = [
-     'so-curator',
      'so-elasticsearch',
      'so-elastic-agent',
      'so-logstash',

salt/logrotate/defaults.yaml

@@ -80,7 +80,7 @@ logrotate:
       - extension .log
       - dateext
       - dateyesterday
-    /opt/so/log/curator/*_x_log:
+    /opt/so/log/elasticsearch/*indices-delete*_x_log:
       - daily
       - rotate 14
       - missingok

salt/logrotate/soc_logrotate.yaml

@@ -56,13 +56,6 @@ logrotate:
       multiline: True
       global: True
       forcedType: "[]string"
-    "/opt/so/log/curator/*_x_log":
-      description: List of logrotate options for this file.
-      title: /opt/so/log/curator/*.log
-      advanced: True
-      multiline: True
-      global: True
-      forcedType: "[]string"
     "/opt/so/log/elasticfleet/*_x_log":
       description: List of logrotate options for this file.
       title: /opt/so/log/elasticfleet/*.log
@@ -77,6 +70,13 @@ logrotate:
       multiline: True
       global: True
       forcedType: "[]string"
+    "/opt/so/log/elasticsearch/*indices-delete*_x_log":
+      description: List of logrotate options for this file.
+      title: /opt/so/log/elasticsearch/*indices-delete.log
+      advanced: True
+      multiline: True
+      global: True
+      forcedType: "[]string"
     "/opt/so/log/suricata/*_x_log":
       description: List of logrotate options for this file.
       title: /opt/so/log/suricata/*.log

salt/manager/tools/sbin/so-elastic-auth-password-reset

@@ -97,8 +97,6 @@ function soUserSync() {
   salt -C 'G@role:so-standalone or G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managersearch or G@role:so-searchnode or G@role:so-heavynode' state.apply logstash queue=True
   printf "\nApplying kibana state to the appropriate nodes.\n\n"
   salt -C 'G@role:so-standalone or G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managersearch' state.apply kibana queue=True
-  printf "\nApplying curator state to the appropriate nodes.\n\n"
-  salt -C 'G@role:so-standalone or G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managersearch or G@role:so-searchnode or G@role:so-heavynode' state.apply curator queue=True
 }
 
 function highstateManager() {

salt/manager/tools/sbin/so-minion

@@ -302,13 +302,6 @@ function add_strelka_to_minion() {
 	"  " >> $PILLARFILE
 }
 
-function add_curator_to_minion() {
-    printf '%s\n'\
-    "curator:"\
-    "  enabled: True"\
-	"  " >> $PILLARFILE
-}
-
 function add_telegraf_to_minion() {
     printf '%s\n'\
     "telegraf:"\
@@ -435,7 +428,6 @@ function createEVAL() {
 	add_playbook_to_minion
 	add_elastalert_to_minion
 	add_kibana_to_minion
-	add_curator_to_minion
 	add_telegraf_to_minion
 	add_influxdb_to_minion
 	add_nginx_to_minion
@@ -458,7 +450,6 @@ function createSTANDALONE() {
 	add_elastalert_to_minion
 	add_kibana_to_minion
 	add_redis_to_minion
-	add_curator_to_minion
 	add_telegraf_to_minion
 	add_influxdb_to_minion
 	add_nginx_to_minion
@@ -478,7 +469,6 @@ function createMANAGER() {
 	add_elastalert_to_minion
 	add_kibana_to_minion
 	add_redis_to_minion
-	add_curator_to_minion
 	add_telegraf_to_minion
 	add_influxdb_to_minion
 	add_nginx_to_minion
@@ -498,7 +488,6 @@ function createMANAGERSEARCH() {
 	add_elastalert_to_minion
 	add_kibana_to_minion
 	add_redis_to_minion
-	add_curator_to_minion
 	add_telegraf_to_minion
 	add_influxdb_to_minion
 	add_nginx_to_minion
@@ -548,7 +537,6 @@ function createHEAVYNODE() {
 	add_sensor_to_minion
 	add_strelka_to_minion
 	add_redis_to_minion
-	add_curator_to_minion
 	add_telegraf_to_minion
 }
 

salt/manager/tools/sbin/soup

@@ -465,9 +465,12 @@ post_to_2.4.30() {
 }
 
 post_to_2.4.40() {
-  echo "Removing Curator close and delete files"
+  echo "Stopping Curator"
-  rm -f /opt/so/conf/curator/action/*-close.y*ml
+  so-curator-stop
-  rm -f /opt/so/conf/curator/action/logs*-delete.y*ml
+  echo "Removing Curator configuration"
+  rm -rf /opt/so/conf/curator/
+  rm -f /usr/sbin/so-curator-*
+  sed -i '/so-curator/d' /opt/so/conf/so-status/so-status.conf
 }
 
 repo_sync() {

salt/manager/tools/sbin_jinja/so-elastic-fleet-reset

@@ -18,7 +18,7 @@ echo "This includes data previously ingested with Fleet such as Zeek and Suricat
 echo "Deployed Elastic Agents will no longer be enrolled and will need to be reinstalled."
 echo "This script should only be used as a last resort to reinstall Elastic Fleet." 
 echo
-echo "If you would like to proceed, type AGREE and hit ENTER."
+echo "If you would like to proceed, then type AGREE and press ENTER."
 echo
 # Read user input
 read INPUT

salt/soc/defaults.yaml

@@ -470,6 +470,18 @@ soc:
         - rule.action
         - rule.reason
         - network.community_id
+      ':pfsense:':
+        - soc_timestamp
+        - source.ip
+        - source.port
+        - destination.ip
+        - destination.port
+        - network.transport
+        - network.direction
+        - observer.ingress.interface.name
+        - event.action
+        - event.reason
+        - network.community_id
       ':osquery:':
         - soc_timestamp
         - source.ip
@@ -1348,7 +1360,7 @@ soc:
               showSubtitle: true
             - name: Firewall
               description: Firewall events grouped by action
-              query: 'tags:firewall | groupby rule.action'
+              query: 'observer.type:firewall | groupby event.action'
               showSubtitle: true
         dashboards:
           advanced: true
@@ -1551,7 +1563,7 @@ soc:
               query: 'tags:s7* | groupby -sankey event.dataset source.ip destination.ip | groupby event.dataset | groupby source.ip | groupby destination.ip | groupby destination.port'
             - name: Firewall
               description: Firewall logs
-              query: 'tags:firewall | groupby -sankey rule.action interface.name | groupby rule.action | groupby interface.name | groupby network.transport | groupby source.ip | groupby destination.ip | groupby destination.port'
+              query: 'observer.type:firewall | groupby -sankey event.action observer.ingress.interface.name | groupby event.action | groupby observer.ingress.interface.name | groupby network.type | groupby network.transport | groupby source.ip | groupby destination.ip | groupby destination.port'
             - name: VLAN
               description: VLAN (Virtual Local Area Network) tagged logs
               query: '* AND _exists_:network.vlan.id | groupby network.vlan.id | groupby source.ip | groupby -sankey source.ip destination.ip | groupby destination.ip | groupby destination.port | groupby event.dataset | groupby event.module | groupby observer.name | groupby source.geo.country_name | groupby destination.geo.country_name'

salt/soc/enabled.sls

@@ -33,21 +33,26 @@ so-soc:
       - /opt/so/conf/soc/soc_users_roles:/opt/sensoroni/rbac/users_roles:rw
       - /opt/so/conf/soc/queue:/opt/sensoroni/queue:rw
       - /opt/so/saltstack:/opt/so/saltstack:rw
-    - extra_hosts: {{ DOCKER_EXTRA_HOSTS }}
+    - extra_hosts:
-      {% if DOCKER.containers['so-soc'].extra_hosts %}
+    {% for node in DOCKER_EXTRA_HOSTS %}
-        {% for XTRAHOST in DOCKER.containers['so-soc'].extra_hosts %}
+    {%   for hostname, ip in node.items() %}
+      - {{hostname}}:{{ip}}
+    {%   endfor %}
+    {% endfor %}
+    {% if DOCKER.containers['so-soc'].extra_hosts %}
+    {%   for XTRAHOST in DOCKER.containers['so-soc'].extra_hosts %}
       - {{ XTRAHOST }}
-        {% endfor %}
+    {%   endfor %}
-      {% endif %}
+    {% endif %}
     - port_bindings:
-      {% for BINDING in DOCKER.containers['so-soc'].port_bindings %}
+    {% for BINDING in DOCKER.containers['so-soc'].port_bindings %}
       - {{ BINDING }}
-      {% endfor %}
+    {% endfor %}
     {% if DOCKER.containers['so-soc'].extra_env %}
     - environment:
-      {% for XTRAENV in DOCKER.containers['so-soc'].extra_env %}
+    {%   for XTRAENV in DOCKER.containers['so-soc'].extra_env %}
       - {{ XTRAENV }}
-      {% endfor %}
+    {%   endfor %}
     {% endif %}
     - watch:
       - file: /opt/so/conf/soc/*

salt/telegraf/defaults.yaml

@@ -48,6 +48,7 @@ telegraf:
       - redis.sh
       - sostatus.sh
     import:
+      - influxdbsize.sh
       - os.sh
       - sostatus.sh
     sensor:

salt/telegraf/etc/telegraf.conf

@@ -186,7 +186,7 @@
 #
 
 # # Read stats from one or more Elasticsearch servers or clusters
-{%- if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone', 'so-heavynode']   %}
+{%- if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-import']   %}
 [[inputs.elasticsearch]]
   servers = ["https://{{ NODEIP }}:9200"]
   cluster_stats = true

salt/top.sls

@@ -74,7 +74,6 @@ base:
     - suricata
     - zeek
     - strelka
-    - curator
     - elastalert
     - utility
     - soctopus
@@ -104,7 +103,6 @@ base:
     - redis
     - elastic-fleet-package-registry
     - kibana
-    - curator
     - elastalert
     - utility
     - soctopus
@@ -140,7 +138,6 @@ base:
     - suricata
     - zeek
     - strelka
-    - curator
     - elastalert
     - utility
     - soctopus
@@ -179,7 +176,6 @@ base:
     - elasticsearch
     - logstash
     - redis
-    - curator
     - elastic-fleet-package-registry
     - kibana
     - elastalert
@@ -199,7 +195,6 @@ base:
     - elasticsearch
     - logstash
     - redis
-    - curator
     - strelka
     - pcap
     - suricata

setup/so-functions

@@ -1412,7 +1412,7 @@ make_some_dirs() {
 	mkdir -p $local_salt_dir/salt/firewall/portgroups
 	mkdir -p $local_salt_dir/salt/firewall/ports
 
-	for THEDIR in bpf pcap elasticsearch ntp firewall redis backup influxdb strelka sensoroni curator soc soctopus docker zeek suricata nginx telegraf logstash soc manager kratos idstools idh elastalert global;do
+	for THEDIR in bpf pcap elasticsearch ntp firewall redis backup influxdb strelka sensoroni soc soctopus docker zeek suricata nginx telegraf logstash soc manager kratos idstools idh elastalert global;do
 	mkdir -p $local_salt_dir/pillar/$THEDIR
 	touch $local_salt_dir/pillar/$THEDIR/adv_$THEDIR.sls
 	touch $local_salt_dir/pillar/$THEDIR/soc_$THEDIR.sls
@@ -1555,7 +1555,6 @@ reserve_group_ids() {
 	logCmd "groupadd -g 931 logstash"
 	logCmd "groupadd -g 932 kibana"
 	logCmd "groupadd -g 933 elastalert"
-	logCmd "groupadd -g 934 curator"
 	logCmd "groupadd -g 937 zeek"
 	logCmd "groupadd -g 940 suricata"
 	logCmd "groupadd -g 941 stenographer"

setup/so-variables

@@ -112,12 +112,6 @@ export sensoroni_pillar_file
 adv_sensoroni_pillar_file="$local_salt_dir/pillar/sensoroni/adv_sensoroni.sls"
 export adv_sensoroni_pillar_file
 
-curator_pillar_file="$local_salt_dir/pillar/curator/soc_curator.sls"
-export curator_pillar_file
-
-adv_curator_pillar_file="$local_salt_dir/pillar/curator/adv_curator.sls"
-export adv_curator_pillar_file
-
 soctopus_pillar_file="$local_salt_dir/pillar/soctopus/soc_soctopus.sls"
 export soctopus_pillar_file
 

setup/so-verify

@@ -33,7 +33,7 @@ log_has_errors() {
     # Ignore Failed: 0 since that is the salt state output, and we detect state failures
     # via Result: False already.
 
-    # This is ignored for Ubuntu
+    # This is ignored for Ubuntu:
     # Failed to restart snapd.mounts-pre.target: Operation refused, unit snapd.mounts-pre.target
     # may be requested by dependency only (it is configured to refuse manual start/stop).
 
@@ -41,6 +41,10 @@ log_has_errors() {
 
     # "remove failed" is caused by a warning generated by upgrade of libwbclient
 
+    # Exit code 100 failure is likely apt-get running in the background, we wait for it to unlock.
+
+    # Failed to deduce dest mapping appears to occur when a shard isn't yet ready. Temporary.
+    
     grep -E "FAILED|Failed|failed|ERROR|Result: False|Error is not recoverable" "$setup_log" | \
         grep -vE "The Salt Master has cached the public key for this node" | \
         grep -vE "Minion failed to authenticate with the master" | \
@@ -58,6 +62,7 @@ log_has_errors() {
         grep -vE "remove failed" | \
         grep -vE "Failed to restart snapd" | \
         grep -vE "Login Failed Details" | \
+        grep -vE "Failed to deduce dest mappings" | \
         grep -vE "response from daemon: unauthorized" | \
         grep -vE "Reading first line of patchfile" | \
         grep -vE "Command failed with exit code" | \