diff --git a/.github/workflows/contrib.yml b/.github/workflows/contrib.yml index 1cb3b773b..395675b43 100644 --- a/.github/workflows/contrib.yml +++ b/.github/workflows/contrib.yml @@ -11,7 +11,7 @@ jobs: steps: - name: "Contributor Check" if: (github.event.comment.body == 'recheck' || github.event.comment.body == 'I have read the CLA Document and I hereby sign the CLA') || github.event_name == 'pull_request_target' - uses: cla-assistant/github-action@v2.1.3-beta + uses: cla-assistant/github-action@v2.3.1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} PERSONAL_ACCESS_TOKEN : ${{ secrets.PERSONAL_ACCESS_TOKEN }} diff --git a/pillar/top.sls b/pillar/top.sls index 53ec8a330..ed3e49254 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -61,8 +61,6 @@ base: - elastalert.adv_elastalert - backup.soc_backup - backup.adv_backup - - curator.soc_curator - - curator.adv_curator - soctopus.soc_soctopus - soctopus.adv_soctopus - minions.{{ grains.id }} @@ -113,8 +111,6 @@ base: - kibana.adv_kibana - strelka.soc_strelka - strelka.adv_strelka - - curator.soc_curator - - curator.adv_curator - kratos.soc_kratos - kratos.adv_kratos - redis.soc_redis @@ -172,8 +168,6 @@ base: - kibana.adv_kibana - strelka.soc_strelka - strelka.adv_strelka - - curator.soc_curator - - curator.adv_curator - backup.soc_backup - backup.adv_backup - zeek.soc_zeek @@ -194,8 +188,6 @@ base: - logstash.adv_logstash - elasticsearch.soc_elasticsearch - elasticsearch.adv_elasticsearch - - curator.soc_curator - - curator.adv_curator - redis.soc_redis - redis.adv_redis - zeek.soc_zeek @@ -268,8 +260,6 @@ base: - soctopus.adv_soctopus - kibana.soc_kibana - kibana.adv_kibana - - curator.soc_curator - - curator.adv_curator - backup.soc_backup - backup.adv_backup - kratos.soc_kratos diff --git a/salt/allowed_states.map.jinja b/salt/allowed_states.map.jinja index 4e3e57f9c..6585bd96c 100644 --- a/salt/allowed_states.map.jinja +++ b/salt/allowed_states.map.jinja @@ -219,10 +219,6 @@ {% do allowed_states.append('kibana.secrets') %} {% endif %} - {% if grains.role in ['so-eval', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-manager'] %} - {% do allowed_states.append('curator') %} - {% endif %} - {% if grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-managersearch'] %} {% do allowed_states.append('elastalert') %} {% endif %} diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common index 7e510e3ad..7900b3c52 100755 --- a/salt/common/tools/sbin/so-image-common +++ b/salt/common/tools/sbin/so-image-common @@ -42,7 +42,6 @@ container_list() { ) elif [ $MANAGERCHECK != 'so-helix' ]; then TRUSTED_CONTAINERS=( - "so-curator" "so-elastalert" "so-elastic-agent" "so-elastic-agent-builder" diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check index d2582ff94..282411ecc 100755 --- a/salt/common/tools/sbin/so-log-check +++ b/salt/common/tools/sbin/so-log-check @@ -144,6 +144,7 @@ if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|status 200" # false positive (request successful, contained error string in content) EXCLUDED_ERRORS="$EXCLUDED_ERRORS|app_layer.error" # false positive (suricata 7) in stats.log e.g. app_layer.error.imap.parser | Total | 0 EXCLUDED_ERRORS="$EXCLUDED_ERRORS|is not an ip string literal" # false positive (Open Canary logging out blank IP addresses) + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|cannot join on an empty table" # false positive (InfluxDB flux query, import nodes) fi if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then diff --git a/salt/common/tools/sbin/so-nsm-clear b/salt/common/tools/sbin/so-nsm-clear index 3d9596238..c9e5e86d2 100755 --- a/salt/common/tools/sbin/so-nsm-clear +++ b/salt/common/tools/sbin/so-nsm-clear @@ -41,8 +41,13 @@ done if [ $SKIP -ne 1 ]; then # Inform user we are about to delete all data echo - echo "This script will delete all NIDS data (PCAP, Suricata, Zeek)" - echo "If you would like to proceed, please type "AGREE" and hit ENTER." + echo "This script will delete all NSM data from /nsm." + echo + echo "This includes Suricata data, Zeek data, and full packet capture (PCAP)." + echo + echo "This will NOT delete any Suricata or Zeek logs that have already been ingested into Elasticsearch." + echo + echo "If you would like to proceed, then type AGREE and press ENTER." echo # Read user input read INPUT @@ -54,8 +59,8 @@ delete_pcap() { [ -d $PCAP_DATA ] && so-pcap-stop && rm -rf $PCAP_DATA/* && so-pcap-start } delete_suricata() { - SURI_LOG="/opt/so/log/suricata/eve.json" - [ -f $SURI_LOG ] && so-suricata-stop && rm -f $SURI_LOG && so-suricata-start + SURI_LOG="/nsm/suricata/" + [ -d $SURI_LOG ] && so-suricata-stop && rm -rf $SURI_LOG/* && so-suricata-start } delete_zeek() { ZEEK_LOG="/nsm/zeek/logs/" diff --git a/salt/curator/config.sls b/salt/curator/config.sls deleted file mode 100644 index 89ff53b2a..000000000 --- a/salt/curator/config.sls +++ /dev/null @@ -1,81 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{% from 'allowed_states.map.jinja' import allowed_states %} -{% if sls.split('.')[0] in allowed_states %} -{% from "curator/map.jinja" import CURATORMERGED %} - -# Create the group -curatorgroup: - group.present: - - name: curator - - gid: 934 - -# Add user -curator: - user.present: - - uid: 934 - - gid: 934 - - home: /opt/so/conf/curator - - createhome: False - -# Create the log directory -curlogdir: - file.directory: - - name: /opt/so/log/curator - - user: 934 - - group: 939 - -curactiondir: - file.directory: - - name: /opt/so/conf/curator/action - - user: 934 - - group: 939 - - makedirs: True - -actionconfs: - file.recurse: - - name: /opt/so/conf/curator/action - - source: salt://curator/files/action - - user: 934 - - group: 939 - - template: jinja - - defaults: - CURATORMERGED: {{ CURATORMERGED.elasticsearch.index_settings }} - -curconf: - file.managed: - - name: /opt/so/conf/curator/curator.yml - - source: salt://curator/files/curator.yml - - user: 934 - - group: 939 - - mode: 660 - - template: jinja - - show_changes: False - -curator_sbin: - file.recurse: - - name: /usr/sbin - - source: salt://curator/tools/sbin - - user: 934 - - group: 939 - - file_mode: 755 - -curator_sbin_jinja: - file.recurse: - - name: /usr/sbin - - source: salt://curator/tools/sbin_jinja - - user: 934 - - group: 939 - - file_mode: 755 - - template: jinja - -{% else %} - -{{sls}}_state_not_allowed: - test.fail_without_changes: - - name: {{sls}}_state_not_allowed - -{% endif %} diff --git a/salt/curator/defaults.yaml b/salt/curator/defaults.yaml deleted file mode 100644 index b2be8d5ed..000000000 --- a/salt/curator/defaults.yaml +++ /dev/null @@ -1,34 +0,0 @@ -curator: - enabled: False - elasticsearch: - index_settings: - so-beats: - delete: 365 - so-elasticsearch: - delete: 365 - so-firewall: - delete: 365 - so-ids: - delete: 365 - so-import: - delete: 73001 - so-kratos: - delete: 365 - so-kibana: - delete: 365 - so-logstash: - delete: 365 - so-netflow: - delete: 365 - so-osquery: - delete: 365 - so-ossec: - delete: 365 - so-redis: - delete: 365 - so-strelka: - delete: 365 - so-syslog: - delete: 365 - so-zeek: - delete: 365 diff --git a/salt/curator/disabled.sls b/salt/curator/disabled.sls deleted file mode 100644 index acf9e3701..000000000 --- a/salt/curator/disabled.sls +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{% from 'allowed_states.map.jinja' import allowed_states %} -{% if sls.split('.')[0] in allowed_states %} - -include: - - curator.sostatus - -so-curator: - docker_container.absent: - - force: True - -so-curator_so-status.disabled: - file.comment: - - name: /opt/so/conf/so-status/so-status.conf - - regex: ^so-curator$ - -so-curator-cluster-close: - cron.absent: - - identifier: so-curator-cluster-close - -so-curator-cluster-delete: - cron.absent: - - identifier: so-curator-cluster-delete - -{% else %} - -{{sls}}_state_not_allowed: - test.fail_without_changes: - - name: {{sls}}_state_not_allowed - -{% endif %} diff --git a/salt/curator/enabled.sls b/salt/curator/enabled.sls deleted file mode 100644 index 916aa920d..000000000 --- a/salt/curator/enabled.sls +++ /dev/null @@ -1,81 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{% from 'allowed_states.map.jinja' import allowed_states %} -{% if sls.split('.')[0] in allowed_states %} -{% from 'vars/globals.map.jinja' import GLOBALS %} -{% from 'docker/docker.map.jinja' import DOCKER %} - -include: - - curator.config - - curator.sostatus - -so-curator: - docker_container.running: - - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-curator:{{ GLOBALS.so_version }} - - start: True - - hostname: curator - - name: so-curator - - user: curator - - networks: - - sobridge: - - ipv4_address: {{ DOCKER.containers['so-curator'].ip }} - - interactive: True - - tty: True - - binds: - - /opt/so/conf/curator/curator.yml:/etc/curator/config/curator.yml:ro - - /opt/so/conf/curator/action/:/etc/curator/action:ro - - /opt/so/log/curator:/var/log/curator:rw - {% if DOCKER.containers['so-curator'].custom_bind_mounts %} - {% for BIND in DOCKER.containers['so-curator'].custom_bind_mounts %} - - {{ BIND }} - {% endfor %} - {% endif %} - {% if DOCKER.containers['so-curator'].extra_hosts %} - - extra_hosts: - {% for XTRAHOST in DOCKER.containers['so-curator'].extra_hosts %} - - {{ XTRAHOST }} - {% endfor %} - {% endif %} - {% if DOCKER.containers['so-curator'].extra_env %} - - environment: - {% for XTRAENV in DOCKER.containers['so-curator'].extra_env %} - - {{ XTRAENV }} - {% endfor %} - {% endif %} - - require: - - file: actionconfs - - file: curconf - - file: curlogdir - - watch: - - file: curconf - -delete_so-curator_so-status.disabled: - file.uncomment: - - name: /opt/so/conf/so-status/so-status.conf - - regex: ^so-curator$ - -so-curator-cluster-close: - cron.absent: - - identifier: so-curator-cluster-close - -so-curator-cluster-delete: - cron.present: - - name: /usr/sbin/so-curator-cluster-delete > /opt/so/log/curator/cron-cluster-delete.log 2>&1 - - identifier: so-curator-cluster-delete - - user: root - - minute: '*/5' - - hour: '*' - - daymonth: '*' - - month: '*' - - dayweek: '*' - -{% else %} - -{{sls}}_state_not_allowed: - test.fail_without_changes: - - name: {{sls}}_state_not_allowed - -{% endif %} diff --git a/salt/curator/files/action/delete.yml b/salt/curator/files/action/delete.yml deleted file mode 100644 index 253c6fd67..000000000 --- a/salt/curator/files/action/delete.yml +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{% import_yaml 'elasticsearch/defaults.yaml' as ELASTICDEFAULTS %} -{% set ELASTICMERGED = salt['pillar.get']('elasticsearch:retention', ELASTICDEFAULTS.elasticsearch.retention, merge=true) %} - -{{ ELASTICMERGED.retention_pct }} - -{%- set log_size_limit = salt['pillar.get']('elasticsearch:log_size_limit') %} -actions: - 1: - action: delete_indices - description: >- - Delete indices when {{log_size_limit}}(GB) is exceeded. - options: - allow_ilm_indices: True - ignore_empty_list: True - disable_action: False - filters: - - filtertype: pattern - kind: regex - value: '^(logstash-.*|so-.*|.ds-logs-.*-so.*)$' - - filtertype: pattern - kind: regex - value: '^(so-case.*)$' - exclude: True - - filtertype: space - source: creation_date - use_age: True - disk_space: {{log_size_limit}} diff --git a/salt/curator/files/action/so-beats-delete.yml b/salt/curator/files/action/so-beats-delete.yml deleted file mode 100644 index c4e1f8b4e..000000000 --- a/salt/curator/files/action/so-beats-delete.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{%- set DELETE_DAYS = CURATORMERGED['so-beats'].delete %} -actions: - 1: - action: delete_indices - description: >- - Delete beats indices when older than {{ DELETE_DAYS }} days. - options: - ignore_empty_list: True - disable_action: False - filters: - - filtertype: pattern - kind: regex - value: '^(logstash-beats.*|so-beats.*)$' - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: {{ DELETE_DAYS }} - exclude: - - \ No newline at end of file diff --git a/salt/curator/files/action/so-elasticsearch-delete.yml b/salt/curator/files/action/so-elasticsearch-delete.yml deleted file mode 100644 index 3c6bf4aac..000000000 --- a/salt/curator/files/action/so-elasticsearch-delete.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{%- set DELETE_DAYS = CURATORMERGED['so-elasticsearch'].delete %} -actions: - 1: - action: delete_indices - description: >- - Delete elasticsearch indices when older than {{ DELETE_DAYS }} days. - options: - ignore_empty_list: True - disable_action: False - filters: - - filtertype: pattern - kind: regex - value: '^(logstash-elasticsearch.*|so-elasticsearch.*)$' - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: {{ DELETE_DAYS }} - exclude: - - \ No newline at end of file diff --git a/salt/curator/files/action/so-firewall-delete.yml b/salt/curator/files/action/so-firewall-delete.yml deleted file mode 100644 index 5143e2fe9..000000000 --- a/salt/curator/files/action/so-firewall-delete.yml +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - - -{%- set DELETE_DAYS = CURATORMERGED['so-firewall'].delete %} -actions: - 1: - action: delete_indices - description: >- - Delete firewall indices when older than {{ DELETE_DAYS }} days. - options: - ignore_empty_list: True - disable_action: False - filters: - - filtertype: pattern - kind: regex - value: '^(logstash-firewall.*|so-firewall.*)$' - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: {{ DELETE_DAYS }} - exclude: - - \ No newline at end of file diff --git a/salt/curator/files/action/so-ids-delete.yml b/salt/curator/files/action/so-ids-delete.yml deleted file mode 100644 index 6cf120fef..000000000 --- a/salt/curator/files/action/so-ids-delete.yml +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - - -{%- set DELETE_DAYS = CURATORMERGED['so-ids'].delete %} -actions: - 1: - action: delete_indices - description: >- - Delete IDS indices when older than {{ DELETE_DAYS }} days. - options: - ignore_empty_list: True - disable_action: False - filters: - - filtertype: pattern - kind: regex - value: '^(logstash-ids.*|so-ids.*)$' - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: {{ DELETE_DAYS }} - exclude: - - \ No newline at end of file diff --git a/salt/curator/files/action/so-import-delete.yml b/salt/curator/files/action/so-import-delete.yml deleted file mode 100644 index 36e213b26..000000000 --- a/salt/curator/files/action/so-import-delete.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{%- set DELETE_DAYS = CURATORMERGED['so-import'].delete %} -actions: - 1: - action: delete_indices - description: >- - Delete import indices when older than {{ DELETE_DAYS }} days. - options: - ignore_empty_list: True - disable_action: False - filters: - - filtertype: pattern - kind: regex - value: '^(logstash-import.*|so-import.*)$' - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: {{ DELETE_DAYS }} - exclude: - - \ No newline at end of file diff --git a/salt/curator/files/action/so-kibana-delete.yml b/salt/curator/files/action/so-kibana-delete.yml deleted file mode 100644 index 971a178fe..000000000 --- a/salt/curator/files/action/so-kibana-delete.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{%- set DELETE_DAYS = CURATORMERGED['so-kibana'].delete %} -actions: - 1: - action: delete_indices - description: >- - Delete kibana indices when older than {{ DELETE_DAYS }} days. - options: - ignore_empty_list: True - disable_action: False - filters: - - filtertype: pattern - kind: regex - value: '^(logstash-kibana.*|so-kibana.*)$' - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: {{ DELETE_DAYS }} - exclude: - - \ No newline at end of file diff --git a/salt/curator/files/action/so-kratos-delete.yml b/salt/curator/files/action/so-kratos-delete.yml deleted file mode 100644 index d7cb2c4ad..000000000 --- a/salt/curator/files/action/so-kratos-delete.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{%- set DELETE_DAYS = CURATORMERGED['so-kratos'].delete %} -actions: - 1: - action: delete_indices - description: >- - Delete kratos indices when older than {{ DELETE_DAYS }} days. - options: - ignore_empty_list: True - disable_action: False - filters: - - filtertype: pattern - kind: regex - value: '^(logstash-kratos.*|so-kratos.*)$' - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: {{ DELETE_DAYS }} - exclude: - - \ No newline at end of file diff --git a/salt/curator/files/action/so-logstash-delete.yml b/salt/curator/files/action/so-logstash-delete.yml deleted file mode 100644 index 1ca1a6f6c..000000000 --- a/salt/curator/files/action/so-logstash-delete.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{%- set DELETE_DAYS = CURATORMERGED['so-logstash'].delete %} -actions: - 1: - action: delete_indices - description: >- - Delete logstash indices when older than {{ DELETE_DAYS }} days. - options: - ignore_empty_list: True - disable_action: False - filters: - - filtertype: pattern - kind: regex - value: '^(logstash-logstash.*|so-logstash.*)$' - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: {{ DELETE_DAYS }} - exclude: - - \ No newline at end of file diff --git a/salt/curator/files/action/so-netflow-delete.yml b/salt/curator/files/action/so-netflow-delete.yml deleted file mode 100644 index 63adaa393..000000000 --- a/salt/curator/files/action/so-netflow-delete.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{%- set DELETE_DAYS = CURATORMERGED['so-netflow'].delete %} -actions: - 1: - action: delete_indices - description: >- - Delete netflow indices when older than {{ DELETE_DAYS }} days. - options: - ignore_empty_list: True - disable_action: False - filters: - - filtertype: pattern - kind: regex - value: '^(logstash-netflow.*|so-netflow.*)$' - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: {{ DELETE_DAYS }} - exclude: - - \ No newline at end of file diff --git a/salt/curator/files/action/so-osquery-delete.yml b/salt/curator/files/action/so-osquery-delete.yml deleted file mode 100644 index b6263b0e8..000000000 --- a/salt/curator/files/action/so-osquery-delete.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{%- set DELETE_DAYS = CURATORMERGED['so-osquery'].delete %} -actions: - 1: - action: delete_indices - description: >- - Delete import indices when older than {{ DELETE_DAYS }} days. - options: - ignore_empty_list: True - disable_action: False - filters: - - filtertype: pattern - kind: regex - value: '^(logstash-osquery.*|so-osquery.*)$' - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: {{ DELETE_DAYS }} - exclude: - - diff --git a/salt/curator/files/action/so-ossec-delete.yml b/salt/curator/files/action/so-ossec-delete.yml deleted file mode 100644 index e24fe3819..000000000 --- a/salt/curator/files/action/so-ossec-delete.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{%- set DELETE_DAYS = CURATORMERGED['so-ossec'].delete %} -actions: - 1: - action: delete_indices - description: >- - Delete ossec indices when older than {{ DELETE_DAYS }} days. - options: - ignore_empty_list: True - disable_action: False - filters: - - filtertype: pattern - kind: regex - value: '^(logstash-ossec.*|so-ossec.*)$' - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: {{ DELETE_DAYS }} - exclude: - - \ No newline at end of file diff --git a/salt/curator/files/action/so-redis-delete.yml b/salt/curator/files/action/so-redis-delete.yml deleted file mode 100644 index 1c7f95ded..000000000 --- a/salt/curator/files/action/so-redis-delete.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{%- set DELETE_DAYS = CURATORMERGED['so-redis'].delete %} -actions: - 1: - action: delete_indices - description: >- - Delete redis indices when older than {{ DELETE_DAYS }} days. - options: - ignore_empty_list: True - disable_action: False - filters: - - filtertype: pattern - kind: regex - value: '^(logstash-redis.*|so-redis.*)$' - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: {{ DELETE_DAYS }} - exclude: - - \ No newline at end of file diff --git a/salt/curator/files/action/so-strelka-delete.yml b/salt/curator/files/action/so-strelka-delete.yml deleted file mode 100644 index 90cf88e46..000000000 --- a/salt/curator/files/action/so-strelka-delete.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{%- set DELETE_DAYS = CURATORMERGED['so-strelka'].delete %} -actions: - 1: - action: delete_indices - description: >- - Delete Strelka indices when older than {{ DELETE_DAYS }} days. - options: - ignore_empty_list: True - disable_action: False - filters: - - filtertype: pattern - kind: regex - value: '^(logstash-strelka.*|so-strelka.*)$' - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: {{ DELETE_DAYS }} - exclude: - - \ No newline at end of file diff --git a/salt/curator/files/action/so-syslog-delete.yml b/salt/curator/files/action/so-syslog-delete.yml deleted file mode 100644 index c11d2ef5a..000000000 --- a/salt/curator/files/action/so-syslog-delete.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{%- set DELETE_DAYS = CURATORMERGED['so-syslog'].delete %} -actions: - 1: - action: delete_indices - description: >- - Delete syslog indices when older than {{ DELETE_DAYS }} days. - options: - ignore_empty_list: True - disable_action: False - filters: - - filtertype: pattern - kind: regex - value: '^(logstash-syslog.*|so-syslog.*)$' - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: {{ DELETE_DAYS }} - exclude: - - \ No newline at end of file diff --git a/salt/curator/files/action/so-zeek-delete.yml b/salt/curator/files/action/so-zeek-delete.yml deleted file mode 100644 index 1f8522696..000000000 --- a/salt/curator/files/action/so-zeek-delete.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{%- set DELETE_DAYS = CURATORMERGED['so-zeek'].delete %} -actions: - 1: - action: delete_indices - description: >- - Delete Zeek indices when older than {{ DELETE_DAYS }} days. - options: - ignore_empty_list: True - disable_action: False - filters: - - filtertype: pattern - kind: regex - value: '^(logstash-zeek.*|so-zeek.*)$' - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: {{ DELETE_DAYS }} - exclude: - - \ No newline at end of file diff --git a/salt/curator/files/curator.yml b/salt/curator/files/curator.yml deleted file mode 100644 index 4ea1dddf7..000000000 --- a/salt/curator/files/curator.yml +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{% from 'vars/globals.map.jinja' import GLOBALS %} -{% if GLOBALS.role in ['so-searchnode', 'so-heavynode'] %} - {%- set elasticsearch = GLOBALS.node_ip -%} -{% elif GLOBALS.role in ['so-eval', 'so-managersearch', 'so-standalone', 'so-manager'] %} - {%- set elasticsearch = GLOBALS.manager_ip -%} -{%- endif %} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} - ---- -# Remember, leave a key empty if there is no value. None will be a string, -# not a Python "NoneType" -elasticsearch: - client: - hosts: - - https://{{elasticsearch}}:9200 - cloud_id: - ca_certs: - client_cert: - client_key: - verify_certs: False - request_timeout: 30 - other_settings: - api_key: - id: - api_key: - master_only: False - username: "{{ ES_USER }}" - password: "{{ ES_PASS }}" - -logging: - loglevel: INFO - logfile: '/var/log/curator/curator.log' - logformat: default - blacklist: ['elasticsearch', 'urllib3'] diff --git a/salt/curator/init.sls b/salt/curator/init.sls deleted file mode 100644 index 201195b60..000000000 --- a/salt/curator/init.sls +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{% from 'curator/map.jinja' import CURATORMERGED %} - -include: -{% if CURATORMERGED.enabled %} - - curator.enabled -{% else %} - - curator.disabled -{% endif %} diff --git a/salt/curator/map.jinja b/salt/curator/map.jinja deleted file mode 100644 index 517209635..000000000 --- a/salt/curator/map.jinja +++ /dev/null @@ -1,7 +0,0 @@ -{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one - or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at - https://securityonion.net/license; you may not use this file except in compliance with the - Elastic License 2.0. #} - -{% import_yaml 'curator/defaults.yaml' as CURATORDEFAULTS %} -{% set CURATORMERGED = salt['pillar.get']('curator', CURATORDEFAULTS.curator, merge=true) %} diff --git a/salt/curator/soc_curator.yaml b/salt/curator/soc_curator.yaml deleted file mode 100644 index acfba7f85..000000000 --- a/salt/curator/soc_curator.yaml +++ /dev/null @@ -1,39 +0,0 @@ -curator: - enabled: - description: You can enable or disable Curator. - helpLink: curator.html - elasticsearch: - index_settings: - so-beats: - delete: &delete - description: Age, in days, when Curator deletes the index. - helpLink: curator.html - forcedType: int - so-elasticsearch: - delete: *delete - so-firewall: - delete: *delete - so-ids: - delete: *delete - so-import: - delete: *delete - so-kratos: - delete: *delete - so-kibana: - delete: *delete - so-logstash: - delete: *delete - so-netflow: - delete: *delete - so-osquery: - delete: *delete - so-ossec: - delete: *delete - so-redis: - delete: *delete - so-strelka: - delete: *delete - so-syslog: - delete: *delete - so-zeek: - delete: *delete diff --git a/salt/curator/sostatus.sls b/salt/curator/sostatus.sls deleted file mode 100644 index de6459a6d..000000000 --- a/salt/curator/sostatus.sls +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{% from 'allowed_states.map.jinja' import allowed_states %} -{% if sls.split('.')[0] in allowed_states %} - -append_so-curator_so-status.conf: - file.append: - - name: /opt/so/conf/so-status/so-status.conf - - text: so-curator - - unless: grep -q so-curator /opt/so/conf/so-status/so-status.conf - -{% else %} - -{{sls}}_state_not_allowed: - test.fail_without_changes: - - name: {{sls}}_state_not_allowed - -{% endif %} diff --git a/salt/curator/tools/sbin/so-curator-delete b/salt/curator/tools/sbin/so-curator-delete deleted file mode 100644 index 2d128bfdf..000000000 --- a/salt/curator/tools/sbin/so-curator-delete +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - - -APP=delete -lf=/tmp/$APP-pidLockFile -# create empty lock file if none exists -cat /dev/null >> $lf -read lastPID < $lf -# if lastPID is not null and a process with that pid exists , exit -[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit -echo $$ > $lf - -docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/delete.yml > /dev/null 2>&1 diff --git a/salt/curator/tools/sbin/so-curator-restart b/salt/curator/tools/sbin/so-curator-restart deleted file mode 100644 index f57e7b22e..000000000 --- a/salt/curator/tools/sbin/so-curator-restart +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - - - -. /usr/sbin/so-common - -/usr/sbin/so-restart curator $1 diff --git a/salt/curator/tools/sbin/so-curator-start b/salt/curator/tools/sbin/so-curator-start deleted file mode 100644 index c5f0fc4d1..000000000 --- a/salt/curator/tools/sbin/so-curator-start +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - - - -. /usr/sbin/so-common - -/usr/sbin/so-start curator $1 diff --git a/salt/curator/tools/sbin/so-curator-stop b/salt/curator/tools/sbin/so-curator-stop deleted file mode 100644 index 30fb07e4b..000000000 --- a/salt/curator/tools/sbin/so-curator-stop +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - - - -. /usr/sbin/so-common - -/usr/sbin/so-stop curator $1 diff --git a/salt/docker/defaults.yaml b/salt/docker/defaults.yaml index a5d6c5d6d..9a27843ae 100644 --- a/salt/docker/defaults.yaml +++ b/salt/docker/defaults.yaml @@ -159,11 +159,6 @@ docker: custom_bind_mounts: [] extra_hosts: [] extra_env: [] - 'so-curator': - final_octet: 43 - custom_bind_mounts: [] - extra_hosts: [] - extra_env: [] 'so-elastic-fleet-package-registry': final_octet: 44 port_bindings: diff --git a/salt/docker/soc_docker.yaml b/salt/docker/soc_docker.yaml index d227a3e85..850324a9e 100644 --- a/salt/docker/soc_docker.yaml +++ b/salt/docker/soc_docker.yaml @@ -60,7 +60,6 @@ docker: so-strelka-gatekeeper: *dockerOptions so-strelka-coordinator: *dockerOptions so-elastalert: *dockerOptions - so-curator: *dockerOptions so-elastic-fleet-package-registry: *dockerOptions so-idh: *dockerOptions so-elastic-agent: *dockerOptions diff --git a/salt/elasticfleet/defaults.yaml b/salt/elasticfleet/defaults.yaml index 7d3883895..2c0fcb816 100644 --- a/salt/elasticfleet/defaults.yaml +++ b/salt/elasticfleet/defaults.yaml @@ -76,6 +76,7 @@ elasticfleet: - pulse_connect_secure - redis - sentinel_one + - snort - snyk - sonicwall_firewall - sophos @@ -85,9 +86,12 @@ elasticfleet: - tcp - tenable_sc - ti_abusech + - ti_anomali + - ti_cybersixgill - ti_misp - ti_otx - ti_recordedfuture + - ti_threatq - udp - vsphere - windows diff --git a/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/import-zeek-logs.json b/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/import-zeek-logs.json index 0979f98b6..492db03dc 100644 --- a/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/import-zeek-logs.json +++ b/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/import-zeek-logs.json @@ -27,6 +27,7 @@ } } } - } + }, + "force": true } {%- endraw -%} diff --git a/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/zeek-logs.json b/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/zeek-logs.json index 32bff857b..5462dc861 100644 --- a/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/zeek-logs.json +++ b/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/zeek-logs.json @@ -28,6 +28,7 @@ } } } - } + }, + "force": true } {%- endraw -%} diff --git a/salt/elasticfleet/files/integrations-optional/sublime_platform.json b/salt/elasticfleet/files/integrations-optional/sublime_platform.json index 8feedc879..0116ed936 100644 --- a/salt/elasticfleet/files/integrations-optional/sublime_platform.json +++ b/salt/elasticfleet/files/integrations-optional/sublime_platform.json @@ -39,6 +39,7 @@ } } } - } + }, + "force": true } {%- endraw -%} diff --git a/salt/elasticfleet/files/integrations/endpoints-initial/osquery.json b/salt/elasticfleet/files/integrations/endpoints-initial/osquery.json index f1774b905..b3d6c4609 100644 --- a/salt/elasticfleet/files/integrations/endpoints-initial/osquery.json +++ b/salt/elasticfleet/files/integrations/endpoints-initial/osquery.json @@ -16,5 +16,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/endpoints-initial/system-endpoints.json b/salt/elasticfleet/files/integrations/endpoints-initial/system-endpoints.json index b6a21249a..699c07ec3 100644 --- a/salt/elasticfleet/files/integrations/endpoints-initial/system-endpoints.json +++ b/salt/elasticfleet/files/integrations/endpoints-initial/system-endpoints.json @@ -73,5 +73,6 @@ "system-system/metrics": { "enabled": false } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/endpoints-initial/windows-endpoints.json b/salt/elasticfleet/files/integrations/endpoints-initial/windows-endpoints.json index b17986a53..8e9813741 100644 --- a/salt/elasticfleet/files/integrations/endpoints-initial/windows-endpoints.json +++ b/salt/elasticfleet/files/integrations/endpoints-initial/windows-endpoints.json @@ -63,5 +63,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/elasticsearch-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/elasticsearch-logs.json index 4c22f92ee..a2aaf5e0a 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/elasticsearch-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/elasticsearch-logs.json @@ -102,5 +102,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/idh-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/idh-logs.json index 29cc1a879..db4b1a8f6 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/idh-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/idh-logs.json @@ -25,5 +25,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json index baa8683ae..3b1a50560 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json @@ -28,5 +28,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/import-suricata-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/import-suricata-logs.json index 3b8cffcc1..c9b036e36 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/import-suricata-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/import-suricata-logs.json @@ -25,5 +25,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/kratos-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/kratos-logs.json index b1fb71077..6a67c9c1c 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/kratos-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/kratos-logs.json @@ -25,5 +25,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/osquery-grid-nodes.json b/salt/elasticfleet/files/integrations/grid-nodes_general/osquery-grid-nodes.json index 0349c9fc3..5527607bd 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/osquery-grid-nodes.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/osquery-grid-nodes.json @@ -16,5 +16,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/redis-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/redis-logs.json index 6b9cbffaf..b822421eb 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/redis-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/redis-logs.json @@ -72,5 +72,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-auth-sync-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-auth-sync-logs.json index 3aa740881..aa39c177b 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-auth-sync-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-auth-sync-logs.json @@ -25,5 +25,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-salt-relay-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-salt-relay-logs.json index 840f36f6b..cc92092e9 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-salt-relay-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-salt-relay-logs.json @@ -25,5 +25,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-sensoroni-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-sensoroni-logs.json index 60ee95f45..61ad057f4 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-sensoroni-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-sensoroni-logs.json @@ -25,5 +25,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-server-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-server-logs.json index b789adc1d..a875e4bfc 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-server-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-server-logs.json @@ -25,5 +25,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/strelka-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/strelka-logs.json index 089b5d4f8..89e9bbe8e 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/strelka-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/strelka-logs.json @@ -25,5 +25,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/suricata-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/suricata-logs.json index a9d857b24..c3b04fd86 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/suricata-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/suricata-logs.json @@ -25,5 +25,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/syslog-tcp-514.json b/salt/elasticfleet/files/integrations/grid-nodes_general/syslog-tcp-514.json index 80baa45ca..4088f5a87 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/syslog-tcp-514.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/syslog-tcp-514.json @@ -28,5 +28,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/syslog-udp-514.json b/salt/elasticfleet/files/integrations/grid-nodes_general/syslog-udp-514.json index 653c788b5..ad32a6964 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/syslog-udp-514.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/syslog-udp-514.json @@ -29,5 +29,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/system-grid-nodes.json b/salt/elasticfleet/files/integrations/grid-nodes_general/system-grid-nodes.json index a5c4c3e81..8e6bf7958 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/system-grid-nodes.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/system-grid-nodes.json @@ -36,5 +36,6 @@ "system-system/metrics": { "enabled": false } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_heavy/osquery-grid-nodes.json b/salt/elasticfleet/files/integrations/grid-nodes_heavy/osquery-grid-nodes.json index b1454d4bd..b1d69a44f 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_heavy/osquery-grid-nodes.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_heavy/osquery-grid-nodes.json @@ -16,5 +16,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_heavy/system-grid-nodes.json b/salt/elasticfleet/files/integrations/grid-nodes_heavy/system-grid-nodes.json index 3df514f0b..6c42086bc 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_heavy/system-grid-nodes.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_heavy/system-grid-nodes.json @@ -36,5 +36,6 @@ "system-system/metrics": { "enabled": false } - } + }, + "force": true } diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 02c2529a6..66916acd1 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -6737,6 +6737,50 @@ elasticsearch: set_priority: priority: 50 min_age: 30d + so-logs-snort_x_log: + index_sorting: False + index_template: + index_patterns: + - "logs-snort.log-*" + template: + settings: + index: + lifecycle: + name: so-logs-snort.log-logs + number_of_replicas: 0 + composed_of: + - "logs-snort.log@package" + - "logs-snort.log@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d so-logs-snyk_x_audit: index_sorting: false index_template: @@ -7622,6 +7666,94 @@ elasticsearch: set_priority: priority: 50 min_age: 30d + so-logs-ti_anomali_x_threatstream: + index_sorting: False + index_template: + index_patterns: + - "logs-ti_anomali.threatstream-*" + template: + settings: + index: + lifecycle: + name: so-logs-ti_anomali.threatstream-logs + number_of_replicas: 0 + composed_of: + - "logs-ti_anomali.threatstream@package" + - "logs-ti_anomali.threatstream@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-ti_cybersixgill_x_threat: + index_sorting: False + index_template: + index_patterns: + - "logs-ti_cybersixgill.threat-*" + template: + settings: + index: + lifecycle: + name: so-logs-ti_cybersixgill.threat-logs + number_of_replicas: 0 + composed_of: + - "logs-ti_cybersixgill.threat@package" + - "logs-ti_cybersixgill.threat@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d so-logs-ti_misp_x_threat: index_sorting: false index_template: @@ -7842,6 +7974,50 @@ elasticsearch: set_priority: priority: 50 min_age: 30d + so-logs-ti_threatq_x_threat: + index_sorting: False + index_template: + index_patterns: + - "logs-ti_threatq.threat-*" + template: + settings: + index: + lifecycle: + name: so-logs-ti_threatq.threat-logs + number_of_replicas: 0 + composed_of: + - "logs-ti_threatq.threat@package" + - "logs-ti_threatq.threat@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d so-logs-vsphere_x_log: index_sorting: False index_template: diff --git a/salt/elasticsearch/disabled.sls b/salt/elasticsearch/disabled.sls index 210ad59a4..2453f9f77 100644 --- a/salt/elasticsearch/disabled.sls +++ b/salt/elasticsearch/disabled.sls @@ -18,6 +18,10 @@ so-elasticsearch_so-status.disabled: - name: /opt/so/conf/so-status/so-status.conf - regex: ^so-elasticsearch$ +so-elasticsearch-indices-delete: + cron.absent: + - identifier: so-elasticsearch-indices-delete + {% else %} {{sls}}_state_not_allowed: diff --git a/salt/elasticsearch/enabled.sls b/salt/elasticsearch/enabled.sls index f7ab7749f..818b0666c 100644 --- a/salt/elasticsearch/enabled.sls +++ b/salt/elasticsearch/enabled.sls @@ -26,7 +26,12 @@ so-elasticsearch: - networks: - sobridge: - ipv4_address: {{ DOCKER.containers['so-elasticsearch'].ip }} - - extra_hosts: {{ LOGSTASH_NODES }} + - extra_hosts: + {% for node in LOGSTASH_NODES %} + {% for hostname, ip in node.items() %} + - {{hostname}}:{{ip}} + {% endfor %} + {% endfor %} {% if DOCKER.containers['so-elasticsearch'].extra_hosts %} {% for XTRAHOST in DOCKER.containers['so-elasticsearch'].extra_hosts %} - {{ XTRAHOST }} @@ -195,6 +200,26 @@ so-elasticsearch-roles-load: - require: - docker_container: so-elasticsearch - file: elasticsearch_sbin_jinja +{% if grains.role in ['so-eval', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-manager'] %} +so-curator-cluster-close: + cron.absent: + - identifier: so-curator-cluster-close + +so-curator-cluster-delete: + cron.absent: + - identifier: so-curator-cluster-delete + +so-elasticsearch-indices-delete: + cron.present: + - name: /usr/sbin/so-elasticsearch-indices-delete > /opt/so/log/elasticsearch/cron-elasticsearch-indices-delete.log 2>&1 + - identifier: so-elasticsearch-indices-delete + - user: root + - minute: '*/5' + - hour: '*' + - daymonth: '*' + - month: '*' + - dayweek: '*' +{% endif %} {% endif %} {% else %} diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index 9d9b3d294..0b93a6c1a 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -407,16 +407,20 @@ elasticsearch: so-logs-sentinel_one_x_group: *indexSettings so-logs-sentinel_one_x_threat: *indexSettings so-logs-sonicwall_firewall_x_log: *indexSettings + so-logs-snort_x_log: *indexSettings so-logs-symantec_endpoint_x_log: *indexSettings so-logs-ti_abusech_x_malware: *indexSettings so-logs-ti_abusech_x_malwarebazaar: *indexSettings so-logs-ti_abusech_x_threatfox: *indexSettings so-logs-ti_abusech_x_url: *indexSettings + so-logs-ti_anomali_x_threatstream: *indexSettings + so-logs-ti_cybersixgill_x_threat: *indexSettings so-logs-ti_misp_x_threat: *indexSettings so-logs-ti_misp_x_threat_attributes: *indexSettings so-logs-ti_otx_x_threat: *indexSettings so-logs-ti_recordedfuture_x_latest_ioc-template: *indexSettings so-logs-ti_recordedfuture_x_threat: *indexSettings + so-logs-ti_threatq_x_threat: *indexSettings so-logs-zscaler_zia_x_alerts: *indexSettings so-logs-zscaler_zia_x_dns: *indexSettings so-logs-zscaler_zia_x_firewall: *indexSettings diff --git a/salt/elasticsearch/tools/sbin/so-elastic-clear b/salt/elasticsearch/tools/sbin/so-elastic-clear index 085327fc4..af7b91ce2 100755 --- a/salt/elasticsearch/tools/sbin/so-elastic-clear +++ b/salt/elasticsearch/tools/sbin/so-elastic-clear @@ -67,7 +67,7 @@ if [ $SKIP -ne 1 ]; then echo echo "This script will delete all data (documents, indices, etc.) in the Elasticsearch database." echo - echo "If you would like to proceed, please type "AGREE" and hit ENTER." + echo "If you would like to proceed, then type AGREE and press ENTER." echo # Read user input read INPUT diff --git a/salt/curator/tools/sbin/so-curator-cluster-delete b/salt/elasticsearch/tools/sbin/so-elasticsearch-indices-delete similarity index 92% rename from salt/curator/tools/sbin/so-curator-cluster-delete rename to salt/elasticsearch/tools/sbin/so-elasticsearch-indices-delete index 0f7945b78..036ff844f 100755 --- a/salt/curator/tools/sbin/so-curator-cluster-delete +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-indices-delete @@ -14,4 +14,4 @@ read lastPID < $lf [ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit echo $$ > $lf -/usr/sbin/so-curator-cluster-delete-delete +/usr/sbin/so-elasticsearch-indices-delete-delete diff --git a/salt/curator/tools/sbin_jinja/so-curator-cluster-delete-delete b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-indices-delete-delete similarity index 88% rename from salt/curator/tools/sbin_jinja/so-curator-cluster-delete-delete rename to salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-indices-delete-delete index e0c5144bc..07feb36bd 100755 --- a/salt/curator/tools/sbin_jinja/so-curator-cluster-delete-delete +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-indices-delete-delete @@ -9,8 +9,8 @@ {%- set ELASTICSEARCH_HOST = GLOBALS.node_ip -%} {%- set RETENTION = salt['pillar.get']('elasticsearch:retention', ELASTICDEFAULTS.elasticsearch.retention, merge=true) -%} -LOG="/opt/so/log/curator/so-curator-cluster-delete.log" -ALERT_LOG="/opt/so/log/curator/alert.log" +LOG="/opt/so/log/elasticsearch/so-elasticsearch-indices-delete.log" +ALERT_LOG="/opt/so/log/elasticsearch/indices-delete-alert.log" LOG_SIZE_LIMIT_GB=$(/usr/sbin/so-elasticsearch-cluster-space-total {{ RETENTION.retention_pct}}) LOG_SIZE_LIMIT=$(( "$LOG_SIZE_LIMIT_GB" * 1000 * 1000 * 1000 )) ITERATION=0 @@ -51,8 +51,11 @@ while overlimit && [[ $ITERATION -lt $MAX_ITERATIONS ]]; do if [ "${INDEX}" != "${CURRENT_WRITE_INDEX}" ]; then # This should not be a write index, so we should be allowed to delete it printf "\n$(date) - Used disk space exceeds LOG_SIZE_LIMIT (${LOG_SIZE_LIMIT_GB} GB) - Deleting ${INDEX} index...\n" >> ${LOG} - /usr/sbin/so-elasticsearch-query ${INDEX} -XDELETE >> ${LOG} 2>&1 + /usr/sbin/so-elasticsearch-query ${INDEX} -XDELETE >> ${LOG} 2>&1 fi + else + printf "\n$(date) - Used disk space exceeds LOG_SIZE_LIMIT (${LOG_SIZE_LIMIT_GB} GB) - There is only one backing index (${INDEX}). Deleting ${DATASTREAM} data stream...\n" >> ${LOG} + /usr/sbin/so-elasticsearch-query _data_stream/$DATASTREAM -XDELETE >> ${LOG} 2>&1 fi if ! overlimit ; then exit diff --git a/salt/firewall/containers.map.jinja b/salt/firewall/containers.map.jinja index 02e8a4644..0ba2389e9 100644 --- a/salt/firewall/containers.map.jinja +++ b/salt/firewall/containers.map.jinja @@ -2,7 +2,6 @@ {% if GLOBALS.role == 'so-eval' %} {% set NODE_CONTAINERS = [ - 'so-curator', 'so-dockerregistry', 'so-elasticsearch', 'so-elastic-fleet', @@ -25,7 +24,6 @@ {% elif GLOBALS.role == 'so-manager' or GLOBALS.role == 'so-standalone' or GLOBALS.role == 'so-managersearch' %} {% set NODE_CONTAINERS = [ - 'so-curator', 'so-dockerregistry', 'so-elasticsearch', 'so-elastic-fleet', @@ -56,7 +54,6 @@ {% elif GLOBALS.role == 'so-heavynode' %} {% set NODE_CONTAINERS = [ - 'so-curator', 'so-elasticsearch', 'so-elastic-agent', 'so-logstash', diff --git a/salt/logrotate/defaults.yaml b/salt/logrotate/defaults.yaml index 4d6a688e4..7333c78e9 100644 --- a/salt/logrotate/defaults.yaml +++ b/salt/logrotate/defaults.yaml @@ -80,7 +80,7 @@ logrotate: - extension .log - dateext - dateyesterday - /opt/so/log/curator/*_x_log: + /opt/so/log/elasticsearch/*indices-delete*_x_log: - daily - rotate 14 - missingok diff --git a/salt/logrotate/soc_logrotate.yaml b/salt/logrotate/soc_logrotate.yaml index 5e6c78fcc..62aa935c9 100644 --- a/salt/logrotate/soc_logrotate.yaml +++ b/salt/logrotate/soc_logrotate.yaml @@ -56,13 +56,6 @@ logrotate: multiline: True global: True forcedType: "[]string" - "/opt/so/log/curator/*_x_log": - description: List of logrotate options for this file. - title: /opt/so/log/curator/*.log - advanced: True - multiline: True - global: True - forcedType: "[]string" "/opt/so/log/elasticfleet/*_x_log": description: List of logrotate options for this file. title: /opt/so/log/elasticfleet/*.log @@ -77,6 +70,13 @@ logrotate: multiline: True global: True forcedType: "[]string" + "/opt/so/log/elasticsearch/*indices-delete*_x_log": + description: List of logrotate options for this file. + title: /opt/so/log/elasticsearch/*indices-delete.log + advanced: True + multiline: True + global: True + forcedType: "[]string" "/opt/so/log/suricata/*_x_log": description: List of logrotate options for this file. title: /opt/so/log/suricata/*.log diff --git a/salt/manager/tools/sbin/so-elastic-auth-password-reset b/salt/manager/tools/sbin/so-elastic-auth-password-reset index 5bc5230c3..b6bfd8721 100755 --- a/salt/manager/tools/sbin/so-elastic-auth-password-reset +++ b/salt/manager/tools/sbin/so-elastic-auth-password-reset @@ -97,8 +97,6 @@ function soUserSync() { salt -C 'G@role:so-standalone or G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managersearch or G@role:so-searchnode or G@role:so-heavynode' state.apply logstash queue=True printf "\nApplying kibana state to the appropriate nodes.\n\n" salt -C 'G@role:so-standalone or G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managersearch' state.apply kibana queue=True - printf "\nApplying curator state to the appropriate nodes.\n\n" - salt -C 'G@role:so-standalone or G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managersearch or G@role:so-searchnode or G@role:so-heavynode' state.apply curator queue=True } function highstateManager() { diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index ada56dc2a..d5225cc82 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -302,13 +302,6 @@ function add_strelka_to_minion() { " " >> $PILLARFILE } -function add_curator_to_minion() { - printf '%s\n'\ - "curator:"\ - " enabled: True"\ - " " >> $PILLARFILE -} - function add_telegraf_to_minion() { printf '%s\n'\ "telegraf:"\ @@ -435,7 +428,6 @@ function createEVAL() { add_playbook_to_minion add_elastalert_to_minion add_kibana_to_minion - add_curator_to_minion add_telegraf_to_minion add_influxdb_to_minion add_nginx_to_minion @@ -458,7 +450,6 @@ function createSTANDALONE() { add_elastalert_to_minion add_kibana_to_minion add_redis_to_minion - add_curator_to_minion add_telegraf_to_minion add_influxdb_to_minion add_nginx_to_minion @@ -478,7 +469,6 @@ function createMANAGER() { add_elastalert_to_minion add_kibana_to_minion add_redis_to_minion - add_curator_to_minion add_telegraf_to_minion add_influxdb_to_minion add_nginx_to_minion @@ -498,7 +488,6 @@ function createMANAGERSEARCH() { add_elastalert_to_minion add_kibana_to_minion add_redis_to_minion - add_curator_to_minion add_telegraf_to_minion add_influxdb_to_minion add_nginx_to_minion @@ -548,7 +537,6 @@ function createHEAVYNODE() { add_sensor_to_minion add_strelka_to_minion add_redis_to_minion - add_curator_to_minion add_telegraf_to_minion } @@ -679,4 +667,4 @@ case "$OPERATION" in *) usage ;; -esac \ No newline at end of file +esac diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 35c934772..4a0adbc98 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -465,9 +465,12 @@ post_to_2.4.30() { } post_to_2.4.40() { - echo "Removing Curator close and delete files" - rm -f /opt/so/conf/curator/action/*-close.y*ml - rm -f /opt/so/conf/curator/action/logs*-delete.y*ml + echo "Stopping Curator" + so-curator-stop + echo "Removing Curator configuration" + rm -rf /opt/so/conf/curator/ + rm -f /usr/sbin/so-curator-* + sed -i '/so-curator/d' /opt/so/conf/so-status/so-status.conf } repo_sync() { diff --git a/salt/manager/tools/sbin_jinja/so-elastic-fleet-reset b/salt/manager/tools/sbin_jinja/so-elastic-fleet-reset index 564156af9..f1112c723 100644 --- a/salt/manager/tools/sbin_jinja/so-elastic-fleet-reset +++ b/salt/manager/tools/sbin_jinja/so-elastic-fleet-reset @@ -18,7 +18,7 @@ echo "This includes data previously ingested with Fleet such as Zeek and Suricat echo "Deployed Elastic Agents will no longer be enrolled and will need to be reinstalled." echo "This script should only be used as a last resort to reinstall Elastic Fleet." echo -echo "If you would like to proceed, type AGREE and hit ENTER." +echo "If you would like to proceed, then type AGREE and press ENTER." echo # Read user input read INPUT @@ -93,4 +93,4 @@ so-elastic-fleet-setup status "Re-installing Elastic Agent on all Grid Nodes..." salt \* state.apply elasticfleet.install_agent_grid queue=True -status "Elastic Fleet Reset complete...." \ No newline at end of file +status "Elastic Fleet Reset complete...." diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 3655019e4..f440bd57b 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -470,6 +470,18 @@ soc: - rule.action - rule.reason - network.community_id + ':pfsense:': + - soc_timestamp + - source.ip + - source.port + - destination.ip + - destination.port + - network.transport + - network.direction + - observer.ingress.interface.name + - event.action + - event.reason + - network.community_id ':osquery:': - soc_timestamp - source.ip @@ -1348,7 +1360,7 @@ soc: showSubtitle: true - name: Firewall description: Firewall events grouped by action - query: 'tags:firewall | groupby rule.action' + query: 'observer.type:firewall | groupby event.action' showSubtitle: true dashboards: advanced: true @@ -1551,7 +1563,7 @@ soc: query: 'tags:s7* | groupby -sankey event.dataset source.ip destination.ip | groupby event.dataset | groupby source.ip | groupby destination.ip | groupby destination.port' - name: Firewall description: Firewall logs - query: 'tags:firewall | groupby -sankey rule.action interface.name | groupby rule.action | groupby interface.name | groupby network.transport | groupby source.ip | groupby destination.ip | groupby destination.port' + query: 'observer.type:firewall | groupby -sankey event.action observer.ingress.interface.name | groupby event.action | groupby observer.ingress.interface.name | groupby network.type | groupby network.transport | groupby source.ip | groupby destination.ip | groupby destination.port' - name: VLAN description: VLAN (Virtual Local Area Network) tagged logs query: '* AND _exists_:network.vlan.id | groupby network.vlan.id | groupby source.ip | groupby -sankey source.ip destination.ip | groupby destination.ip | groupby destination.port | groupby event.dataset | groupby event.module | groupby observer.name | groupby source.geo.country_name | groupby destination.geo.country_name' diff --git a/salt/soc/enabled.sls b/salt/soc/enabled.sls index 9e588e5b1..2661587f4 100644 --- a/salt/soc/enabled.sls +++ b/salt/soc/enabled.sls @@ -33,21 +33,26 @@ so-soc: - /opt/so/conf/soc/soc_users_roles:/opt/sensoroni/rbac/users_roles:rw - /opt/so/conf/soc/queue:/opt/sensoroni/queue:rw - /opt/so/saltstack:/opt/so/saltstack:rw - - extra_hosts: {{ DOCKER_EXTRA_HOSTS }} - {% if DOCKER.containers['so-soc'].extra_hosts %} - {% for XTRAHOST in DOCKER.containers['so-soc'].extra_hosts %} + - extra_hosts: + {% for node in DOCKER_EXTRA_HOSTS %} + {% for hostname, ip in node.items() %} + - {{hostname}}:{{ip}} + {% endfor %} + {% endfor %} + {% if DOCKER.containers['so-soc'].extra_hosts %} + {% for XTRAHOST in DOCKER.containers['so-soc'].extra_hosts %} - {{ XTRAHOST }} - {% endfor %} - {% endif %} + {% endfor %} + {% endif %} - port_bindings: - {% for BINDING in DOCKER.containers['so-soc'].port_bindings %} + {% for BINDING in DOCKER.containers['so-soc'].port_bindings %} - {{ BINDING }} - {% endfor %} + {% endfor %} {% if DOCKER.containers['so-soc'].extra_env %} - environment: - {% for XTRAENV in DOCKER.containers['so-soc'].extra_env %} + {% for XTRAENV in DOCKER.containers['so-soc'].extra_env %} - {{ XTRAENV }} - {% endfor %} + {% endfor %} {% endif %} - watch: - file: /opt/so/conf/soc/* diff --git a/salt/telegraf/defaults.yaml b/salt/telegraf/defaults.yaml index 676f365cf..fa9d9b2b9 100644 --- a/salt/telegraf/defaults.yaml +++ b/salt/telegraf/defaults.yaml @@ -48,6 +48,7 @@ telegraf: - redis.sh - sostatus.sh import: + - influxdbsize.sh - os.sh - sostatus.sh sensor: diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf index 45b1283e0..1c5801645 100644 --- a/salt/telegraf/etc/telegraf.conf +++ b/salt/telegraf/etc/telegraf.conf @@ -186,7 +186,7 @@ # # # Read stats from one or more Elasticsearch servers or clusters -{%- if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone', 'so-heavynode'] %} +{%- if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-import'] %} [[inputs.elasticsearch]] servers = ["https://{{ NODEIP }}:9200"] cluster_stats = true diff --git a/salt/top.sls b/salt/top.sls index 4f84e17ac..5e7d13532 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -74,7 +74,6 @@ base: - suricata - zeek - strelka - - curator - elastalert - utility - soctopus @@ -104,7 +103,6 @@ base: - redis - elastic-fleet-package-registry - kibana - - curator - elastalert - utility - soctopus @@ -140,7 +138,6 @@ base: - suricata - zeek - strelka - - curator - elastalert - utility - soctopus @@ -179,7 +176,6 @@ base: - elasticsearch - logstash - redis - - curator - elastic-fleet-package-registry - kibana - elastalert @@ -199,7 +195,6 @@ base: - elasticsearch - logstash - redis - - curator - strelka - pcap - suricata diff --git a/setup/so-functions b/setup/so-functions index 6ad0947d1..e1faa275e 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1412,7 +1412,7 @@ make_some_dirs() { mkdir -p $local_salt_dir/salt/firewall/portgroups mkdir -p $local_salt_dir/salt/firewall/ports - for THEDIR in bpf pcap elasticsearch ntp firewall redis backup influxdb strelka sensoroni curator soc soctopus docker zeek suricata nginx telegraf logstash soc manager kratos idstools idh elastalert global;do + for THEDIR in bpf pcap elasticsearch ntp firewall redis backup influxdb strelka sensoroni soc soctopus docker zeek suricata nginx telegraf logstash soc manager kratos idstools idh elastalert global;do mkdir -p $local_salt_dir/pillar/$THEDIR touch $local_salt_dir/pillar/$THEDIR/adv_$THEDIR.sls touch $local_salt_dir/pillar/$THEDIR/soc_$THEDIR.sls @@ -1555,7 +1555,6 @@ reserve_group_ids() { logCmd "groupadd -g 931 logstash" logCmd "groupadd -g 932 kibana" logCmd "groupadd -g 933 elastalert" - logCmd "groupadd -g 934 curator" logCmd "groupadd -g 937 zeek" logCmd "groupadd -g 940 suricata" logCmd "groupadd -g 941 stenographer" diff --git a/setup/so-variables b/setup/so-variables index 7f6522487..511dfc43b 100644 --- a/setup/so-variables +++ b/setup/so-variables @@ -112,12 +112,6 @@ export sensoroni_pillar_file adv_sensoroni_pillar_file="$local_salt_dir/pillar/sensoroni/adv_sensoroni.sls" export adv_sensoroni_pillar_file -curator_pillar_file="$local_salt_dir/pillar/curator/soc_curator.sls" -export curator_pillar_file - -adv_curator_pillar_file="$local_salt_dir/pillar/curator/adv_curator.sls" -export adv_curator_pillar_file - soctopus_pillar_file="$local_salt_dir/pillar/soctopus/soc_soctopus.sls" export soctopus_pillar_file diff --git a/setup/so-verify b/setup/so-verify index 3c20d22b4..6f47940ac 100755 --- a/setup/so-verify +++ b/setup/so-verify @@ -33,13 +33,17 @@ log_has_errors() { # Ignore Failed: 0 since that is the salt state output, and we detect state failures # via Result: False already. - # This is ignored for Ubuntu + # This is ignored for Ubuntu: # Failed to restart snapd.mounts-pre.target: Operation refused, unit snapd.mounts-pre.target # may be requested by dependency only (it is configured to refuse manual start/stop). # Command failed with exit code is output during retry loops. # "remove failed" is caused by a warning generated by upgrade of libwbclient + + # Exit code 100 failure is likely apt-get running in the background, we wait for it to unlock. + + # Failed to deduce dest mapping appears to occur when a shard isn't yet ready. Temporary. grep -E "FAILED|Failed|failed|ERROR|Result: False|Error is not recoverable" "$setup_log" | \ grep -vE "The Salt Master has cached the public key for this node" | \ @@ -58,6 +62,7 @@ log_has_errors() { grep -vE "remove failed" | \ grep -vE "Failed to restart snapd" | \ grep -vE "Login Failed Details" | \ + grep -vE "Failed to deduce dest mappings" | \ grep -vE "response from daemon: unauthorized" | \ grep -vE "Reading first line of patchfile" | \ grep -vE "Command failed with exit code" | \