CSEC_PUBLIC/securityonion
1
0
Fork 0
mirror of https://github.com/Security-Onion-Solutions/securityonion.git synced 2026-04-01 10:21:51 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch '2.4/dev' into fix/receiver

Browse Source
This commit is contained in:
Josh Patterson
2023-12-18 13:55:23 -05:00
committed by GitHub
parent 33a9ac5701 88684a6c19
commit f032ff40a2
84 changed files with 335 additions and 956 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/contrib.yml vendored
View File

@@ -11,7 +11,7 @@ jobs:
steps:
- name: "Contributor Check"
if: (github.event.comment.body == 'recheck' || github.event.comment.body == 'I have read the CLA Document and I hereby sign the CLA') || github.event_name == 'pull_request_target'
uses: cla-assistant/github-action@v2.1.3-beta
uses: cla-assistant/github-action@v2.3.1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
PERSONAL_ACCESS_TOKEN : ${{ secrets.PERSONAL_ACCESS_TOKEN }}

10
pillar/top.sls
View File

@@ -61,8 +61,6 @@ base:
- elastalert.adv_elastalert
- backup.soc_backup
- backup.adv_backup
- curator.soc_curator
- curator.adv_curator
- soctopus.soc_soctopus
- soctopus.adv_soctopus
- minions.{{ grains.id }}
@@ -113,8 +111,6 @@ base:
- kibana.adv_kibana
- strelka.soc_strelka
- strelka.adv_strelka
- curator.soc_curator
- curator.adv_curator
- kratos.soc_kratos
- kratos.adv_kratos
- redis.soc_redis
@@ -172,8 +168,6 @@ base:
- kibana.adv_kibana
- strelka.soc_strelka
- strelka.adv_strelka
- curator.soc_curator
- curator.adv_curator
- backup.soc_backup
- backup.adv_backup
- zeek.soc_zeek
@@ -194,8 +188,6 @@ base:
- logstash.adv_logstash
- elasticsearch.soc_elasticsearch
- elasticsearch.adv_elasticsearch
- curator.soc_curator
- curator.adv_curator
- redis.soc_redis
- redis.adv_redis
- zeek.soc_zeek
@@ -268,8 +260,6 @@ base:
- soctopus.adv_soctopus
- kibana.soc_kibana
- kibana.adv_kibana
- curator.soc_curator
- curator.adv_curator
- backup.soc_backup
- backup.adv_backup
- kratos.soc_kratos

4
salt/allowed_states.map.jinja
View File

@@ -219,10 +219,6 @@
{% do allowed_states.append('kibana.secrets') %}
{% endif %}
{% if grains.role in ['so-eval', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-manager'] %}
{% do allowed_states.append('curator') %}
{% endif %}
{% if grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-managersearch'] %}
{% do allowed_states.append('elastalert') %}
{% endif %}

1
salt/common/tools/sbin/so-image-common
View File

@@ -42,7 +42,6 @@ container_list() {
)
elif [ $MANAGERCHECK != 'so-helix' ]; then
TRUSTED_CONTAINERS=(
"so-curator"
"so-elastalert"
"so-elastic-agent"
"so-elastic-agent-builder"

1
salt/common/tools/sbin/so-log-check
View File

@@ -144,6 +144,7 @@ if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|status 200" # false positive (request successful, contained error string in content)
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|app_layer.error" # false positive (suricata 7) in stats.log e.g. app_layer.error.imap.parser | Total | 0
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|is not an ip string literal" # false positive (Open Canary logging out blank IP addresses)
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|cannot join on an empty table" # false positive (InfluxDB flux query, import nodes)
fi
if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then

13
salt/common/tools/sbin/so-nsm-clear
View File

@@ -41,8 +41,13 @@ done
if [ $SKIP -ne 1 ]; then
# Inform user we are about to delete all data
echo
echo "This script will delete all NIDS data (PCAP, Suricata, Zeek)"
echo "If you would like to proceed, please type "AGREE" and hit ENTER."
echo "This script will delete all NSM data from /nsm."
echo
echo "This includes Suricata data, Zeek data, and full packet capture (PCAP)."
echo
echo "This will NOT delete any Suricata or Zeek logs that have already been ingested into Elasticsearch."
echo
echo "If you would like to proceed, then type AGREE and press ENTER."
echo
# Read user input
read INPUT
@@ -54,8 +59,8 @@ delete_pcap() {
[ -d $PCAP_DATA ] && so-pcap-stop && rm -rf $PCAP_DATA/* && so-pcap-start
}
delete_suricata() {
SURI_LOG="/opt/so/log/suricata/eve.json"
[ -f $SURI_LOG ] && so-suricata-stop && rm -f $SURI_LOG && so-suricata-start
SURI_LOG="/nsm/suricata/"
[ -d $SURI_LOG ] && so-suricata-stop && rm -rf $SURI_LOG/* && so-suricata-start
}
delete_zeek() {
ZEEK_LOG="/nsm/zeek/logs/"

81
salt/curator/config.sls
View File

@@ -1,81 +0,0 @@
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{% from 'allowed_states.map.jinja' import allowed_states %}
{% if sls.split('.')[0] in allowed_states %}
{% from "curator/map.jinja" import CURATORMERGED %}
# Create the group
curatorgroup:
group.present:
- name: curator
- gid: 934
# Add user
curator:
user.present:
- uid: 934
- gid: 934
- home: /opt/so/conf/curator
- createhome: False
# Create the log directory
curlogdir:
file.directory:
- name: /opt/so/log/curator
- user: 934
- group: 939
curactiondir:
file.directory:
- name: /opt/so/conf/curator/action
- user: 934
- group: 939
- makedirs: True
actionconfs:
file.recurse:
- name: /opt/so/conf/curator/action
- source: salt://curator/files/action
- user: 934
- group: 939
- template: jinja
- defaults:
CURATORMERGED: {{ CURATORMERGED.elasticsearch.index_settings }}
curconf:
file.managed:
- name: /opt/so/conf/curator/curator.yml
- source: salt://curator/files/curator.yml
- user: 934
- group: 939
- mode: 660
- template: jinja
- show_changes: False
curator_sbin:
file.recurse:
- name: /usr/sbin
- source: salt://curator/tools/sbin
- user: 934
- group: 939
- file_mode: 755
curator_sbin_jinja:
file.recurse:
- name: /usr/sbin
- source: salt://curator/tools/sbin_jinja
- user: 934
- group: 939
- file_mode: 755
- template: jinja
{% else %}
{{sls}}_state_not_allowed:
test.fail_without_changes:
- name: {{sls}}_state_not_allowed
{% endif %}

34
salt/curator/defaults.yaml
View File

@@ -1,34 +0,0 @@
curator:
enabled: False
elasticsearch:
index_settings:
so-beats:
delete: 365
so-elasticsearch:
delete: 365
so-firewall:
delete: 365
so-ids:
delete: 365
so-import:
delete: 73001
so-kratos:
delete: 365
so-kibana:
delete: 365
so-logstash:
delete: 365
so-netflow:
delete: 365
so-osquery:
delete: 365
so-ossec:
delete: 365
so-redis:
delete: 365
so-strelka:
delete: 365
so-syslog:
delete: 365
so-zeek:
delete: 365

35
salt/curator/disabled.sls
View File

@@ -1,35 +0,0 @@
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{% from 'allowed_states.map.jinja' import allowed_states %}
{% if sls.split('.')[0] in allowed_states %}
include:
- curator.sostatus
so-curator:
docker_container.absent:
- force: True
so-curator_so-status.disabled:
file.comment:
- name: /opt/so/conf/so-status/so-status.conf
- regex: ^so-curator$
so-curator-cluster-close:
cron.absent:
- identifier: so-curator-cluster-close
so-curator-cluster-delete:
cron.absent:
- identifier: so-curator-cluster-delete
{% else %}
{{sls}}_state_not_allowed:
test.fail_without_changes:
- name: {{sls}}_state_not_allowed
{% endif %}

81
salt/curator/enabled.sls
View File

@@ -1,81 +0,0 @@
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{% from 'allowed_states.map.jinja' import allowed_states %}
{% if sls.split('.')[0] in allowed_states %}
{% from 'vars/globals.map.jinja' import GLOBALS %}
{% from 'docker/docker.map.jinja' import DOCKER %}
include:
- curator.config
- curator.sostatus
so-curator:
docker_container.running:
- image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-curator:{{ GLOBALS.so_version }}
- start: True
- hostname: curator
- name: so-curator
- user: curator
- networks:
- sobridge:
- ipv4_address: {{ DOCKER.containers['so-curator'].ip }}
- interactive: True
- tty: True
- binds:
- /opt/so/conf/curator/curator.yml:/etc/curator/config/curator.yml:ro
- /opt/so/conf/curator/action/:/etc/curator/action:ro
- /opt/so/log/curator:/var/log/curator:rw
{% if DOCKER.containers['so-curator'].custom_bind_mounts %}
{% for BIND in DOCKER.containers['so-curator'].custom_bind_mounts %}
- {{ BIND }}
{% endfor %}
{% endif %}
{% if DOCKER.containers['so-curator'].extra_hosts %}
- extra_hosts:
{% for XTRAHOST in DOCKER.containers['so-curator'].extra_hosts %}
- {{ XTRAHOST }}
{% endfor %}
{% endif %}
{% if DOCKER.containers['so-curator'].extra_env %}
- environment:
{% for XTRAENV in DOCKER.containers['so-curator'].extra_env %}
- {{ XTRAENV }}
{% endfor %}
{% endif %}
- require:
- file: actionconfs
- file: curconf
- file: curlogdir
- watch:
- file: curconf
delete_so-curator_so-status.disabled:
file.uncomment:
- name: /opt/so/conf/so-status/so-status.conf
- regex: ^so-curator$
so-curator-cluster-close:
cron.absent:
- identifier: so-curator-cluster-close
so-curator-cluster-delete:
cron.present:
- name: /usr/sbin/so-curator-cluster-delete > /opt/so/log/curator/cron-cluster-delete.log 2>&1
- identifier: so-curator-cluster-delete
- user: root
- minute: '*/5'
- hour: '*'
- daymonth: '*'
- month: '*'
- dayweek: '*'
{% else %}
{{sls}}_state_not_allowed:
test.fail_without_changes:
- name: {{sls}}_state_not_allowed
{% endif %}

32
salt/curator/files/action/delete.yml
View File

@@ -1,32 +0,0 @@
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{% import_yaml 'elasticsearch/defaults.yaml' as ELASTICDEFAULTS %}
{% set ELASTICMERGED = salt['pillar.get']('elasticsearch:retention', ELASTICDEFAULTS.elasticsearch.retention, merge=true) %}
{{ ELASTICMERGED.retention_pct }}
{%- set log_size_limit = salt['pillar.get']('elasticsearch:log_size_limit') %}
actions:
1:
action: delete_indices
description: >-
Delete indices when {{log_size_limit}}(GB) is exceeded.
options:
allow_ilm_indices: True
ignore_empty_list: True
disable_action: False
filters:
- filtertype: pattern
kind: regex
value: '^(logstash-.*|so-.*|.ds-logs-.*-so.*)$'
- filtertype: pattern
kind: regex
value: '^(so-case.*)$'
exclude: True
- filtertype: space
source: creation_date
use_age: True
disk_space: {{log_size_limit}}

27
salt/curator/files/action/so-beats-delete.yml
View File

@@ -1,27 +0,0 @@
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{%- set DELETE_DAYS = CURATORMERGED['so-beats'].delete %}
actions:
1:
action: delete_indices
description: >-
Delete beats indices when older than {{ DELETE_DAYS }} days.
options:
ignore_empty_list: True
disable_action: False
filters:
- filtertype: pattern
kind: regex
value: '^(logstash-beats.*|so-beats.*)$'
- filtertype: age
source: name
direction: older
timestring: '%Y.%m.%d'
unit: days
unit_count: {{ DELETE_DAYS }}
exclude:

27
salt/curator/files/action/so-elasticsearch-delete.yml
View File

@@ -1,27 +0,0 @@
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{%- set DELETE_DAYS = CURATORMERGED['so-elasticsearch'].delete %}
actions:
1:
action: delete_indices
description: >-
Delete elasticsearch indices when older than {{ DELETE_DAYS }} days.
options:
ignore_empty_list: True
disable_action: False
filters:
- filtertype: pattern
kind: regex
value: '^(logstash-elasticsearch.*|so-elasticsearch.*)$'
- filtertype: age
source: name
direction: older
timestring: '%Y.%m.%d'
unit: days
unit_count: {{ DELETE_DAYS }}
exclude:

28
salt/curator/files/action/so-firewall-delete.yml
View File

@@ -1,28 +0,0 @@
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{%- set DELETE_DAYS = CURATORMERGED['so-firewall'].delete %}
actions:
1:
action: delete_indices
description: >-
Delete firewall indices when older than {{ DELETE_DAYS }} days.
options:
ignore_empty_list: True
disable_action: False
filters:
- filtertype: pattern
kind: regex
value: '^(logstash-firewall.*|so-firewall.*)$'
- filtertype: age
source: name
direction: older
timestring: '%Y.%m.%d'
unit: days
unit_count: {{ DELETE_DAYS }}
exclude:

28
salt/curator/files/action/so-ids-delete.yml
View File

@@ -1,28 +0,0 @@
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{%- set DELETE_DAYS = CURATORMERGED['so-ids'].delete %}
actions:
1:
action: delete_indices
description: >-
Delete IDS indices when older than {{ DELETE_DAYS }} days.
options:
ignore_empty_list: True
disable_action: False
filters:
- filtertype: pattern
kind: regex
value: '^(logstash-ids.*|so-ids.*)$'
- filtertype: age
source: name
direction: older
timestring: '%Y.%m.%d'
unit: days
unit_count: {{ DELETE_DAYS }}
exclude:

27
salt/curator/files/action/so-import-delete.yml
View File

@@ -1,27 +0,0 @@
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{%- set DELETE_DAYS = CURATORMERGED['so-import'].delete %}
actions:
1:
action: delete_indices
description: >-
Delete import indices when older than {{ DELETE_DAYS }} days.
options:
ignore_empty_list: True
disable_action: False
filters:
- filtertype: pattern
kind: regex
value: '^(logstash-import.*|so-import.*)$'
- filtertype: age
source: name
direction: older
timestring: '%Y.%m.%d'
unit: days
unit_count: {{ DELETE_DAYS }}
exclude:

27
salt/curator/files/action/so-kibana-delete.yml
View File

@@ -1,27 +0,0 @@
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{%- set DELETE_DAYS = CURATORMERGED['so-kibana'].delete %}
actions:
1:
action: delete_indices
description: >-
Delete kibana indices when older than {{ DELETE_DAYS }} days.
options:
ignore_empty_list: True
disable_action: False
filters:
- filtertype: pattern
kind: regex
value: '^(logstash-kibana.*|so-kibana.*)$'
- filtertype: age
source: name
direction: older
timestring: '%Y.%m.%d'
unit: days
unit_count: {{ DELETE_DAYS }}
exclude:

27
salt/curator/files/action/so-kratos-delete.yml
View File

@@ -1,27 +0,0 @@
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{%- set DELETE_DAYS = CURATORMERGED['so-kratos'].delete %}
actions:
1:
action: delete_indices
description: >-
Delete kratos indices when older than {{ DELETE_DAYS }} days.
options:
ignore_empty_list: True
disable_action: False
filters:
- filtertype: pattern
kind: regex
value: '^(logstash-kratos.*|so-kratos.*)$'
- filtertype: age
source: name
direction: older
timestring: '%Y.%m.%d'
unit: days
unit_count: {{ DELETE_DAYS }}
exclude:

27
salt/curator/files/action/so-logstash-delete.yml
View File

@@ -1,27 +0,0 @@
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{%- set DELETE_DAYS = CURATORMERGED['so-logstash'].delete %}
actions:
1:
action: delete_indices
description: >-
Delete logstash indices when older than {{ DELETE_DAYS }} days.
options:
ignore_empty_list: True
disable_action: False
filters:
- filtertype: pattern
kind: regex
value: '^(logstash-logstash.*|so-logstash.*)$'
- filtertype: age
source: name
direction: older
timestring: '%Y.%m.%d'
unit: days
unit_count: {{ DELETE_DAYS }}
exclude:

27
salt/curator/files/action/so-netflow-delete.yml
View File

@@ -1,27 +0,0 @@
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{%- set DELETE_DAYS = CURATORMERGED['so-netflow'].delete %}
actions:
1:
action: delete_indices
description: >-
Delete netflow indices when older than {{ DELETE_DAYS }} days.
options:
ignore_empty_list: True
disable_action: False
filters:
- filtertype: pattern
kind: regex
value: '^(logstash-netflow.*|so-netflow.*)$'
- filtertype: age
source: name
direction: older
timestring: '%Y.%m.%d'
unit: days
unit_count: {{ DELETE_DAYS }}
exclude:

27
salt/curator/files/action/so-osquery-delete.yml
View File

@@ -1,27 +0,0 @@
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{%- set DELETE_DAYS = CURATORMERGED['so-osquery'].delete %}
actions:
1:
action: delete_indices
description: >-
Delete import indices when older than {{ DELETE_DAYS }} days.
options:
ignore_empty_list: True
disable_action: False
filters:
- filtertype: pattern
kind: regex
value: '^(logstash-osquery.*|so-osquery.*)$'
- filtertype: age
source: name
direction: older
timestring: '%Y.%m.%d'
unit: days
unit_count: {{ DELETE_DAYS }}
exclude:

27
salt/curator/files/action/so-ossec-delete.yml
View File

@@ -1,27 +0,0 @@
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{%- set DELETE_DAYS = CURATORMERGED['so-ossec'].delete %}
actions:
1:
action: delete_indices
description: >-
Delete ossec indices when older than {{ DELETE_DAYS }} days.
options:
ignore_empty_list: True
disable_action: False
filters:
- filtertype: pattern
kind: regex
value: '^(logstash-ossec.*|so-ossec.*)$'
- filtertype: age
source: name
direction: older
timestring: '%Y.%m.%d'
unit: days
unit_count: {{ DELETE_DAYS }}
exclude:

27
salt/curator/files/action/so-redis-delete.yml
View File

@@ -1,27 +0,0 @@
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{%- set DELETE_DAYS = CURATORMERGED['so-redis'].delete %}
actions:
1:
action: delete_indices
description: >-
Delete redis indices when older than {{ DELETE_DAYS }} days.
options:
ignore_empty_list: True
disable_action: False
filters:
- filtertype: pattern
kind: regex
value: '^(logstash-redis.*|so-redis.*)$'
- filtertype: age
source: name
direction: older
timestring: '%Y.%m.%d'
unit: days
unit_count: {{ DELETE_DAYS }}
exclude:

27
salt/curator/files/action/so-strelka-delete.yml
View File

@@ -1,27 +0,0 @@
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{%- set DELETE_DAYS = CURATORMERGED['so-strelka'].delete %}
actions:
1:
action: delete_indices
description: >-
Delete Strelka indices when older than {{ DELETE_DAYS }} days.
options:
ignore_empty_list: True
disable_action: False
filters:
- filtertype: pattern
kind: regex
value: '^(logstash-strelka.*|so-strelka.*)$'
- filtertype: age
source: name
direction: older
timestring: '%Y.%m.%d'
unit: days
unit_count: {{ DELETE_DAYS }}
exclude:

27
salt/curator/files/action/so-syslog-delete.yml
View File

@@ -1,27 +0,0 @@
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{%- set DELETE_DAYS = CURATORMERGED['so-syslog'].delete %}
actions:
1:
action: delete_indices
description: >-
Delete syslog indices when older than {{ DELETE_DAYS }} days.
options:
ignore_empty_list: True
disable_action: False
filters:
- filtertype: pattern
kind: regex
value: '^(logstash-syslog.*|so-syslog.*)$'
- filtertype: age
source: name
direction: older
timestring: '%Y.%m.%d'
unit: days
unit_count: {{ DELETE_DAYS }}
exclude:

27
salt/curator/files/action/so-zeek-delete.yml
View File

@@ -1,27 +0,0 @@
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{%- set DELETE_DAYS = CURATORMERGED['so-zeek'].delete %}
actions:
1:
action: delete_indices
description: >-
Delete Zeek indices when older than {{ DELETE_DAYS }} days.
options:
ignore_empty_list: True
disable_action: False
filters:
- filtertype: pattern
kind: regex
value: '^(logstash-zeek.*|so-zeek.*)$'
- filtertype: age
source: name
direction: older
timestring: '%Y.%m.%d'
unit: days
unit_count: {{ DELETE_DAYS }}
exclude:

40
salt/curator/files/curator.yml
View File

@@ -1,40 +0,0 @@
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{% from 'vars/globals.map.jinja' import GLOBALS %}
{% if GLOBALS.role in ['so-searchnode', 'so-heavynode'] %}
{%- set elasticsearch = GLOBALS.node_ip -%}
{% elif GLOBALS.role in ['so-eval', 'so-managersearch', 'so-standalone', 'so-manager'] %}
{%- set elasticsearch = GLOBALS.manager_ip -%}
{%- endif %}
{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %}
{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %}
---
# Remember, leave a key empty if there is no value. None will be a string,
# not a Python "NoneType"
elasticsearch:
client:
hosts:
- https://{{elasticsearch}}:9200
cloud_id:
ca_certs:
client_cert:
client_key:
verify_certs: False
request_timeout: 30
other_settings:
api_key:
id:
api_key:
master_only: False
username: "{{ ES_USER }}"
password: "{{ ES_PASS }}"
logging:
loglevel: INFO
logfile: '/var/log/curator/curator.log'
logformat: default
blacklist: ['elasticsearch', 'urllib3']

13
salt/curator/init.sls
View File

@@ -1,13 +0,0 @@
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{% from 'curator/map.jinja' import CURATORMERGED %}
include:
{% if CURATORMERGED.enabled %}
- curator.enabled
{% else %}
- curator.disabled
{% endif %}

7
salt/curator/map.jinja
View File

@@ -1,7 +0,0 @@
{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
https://securityonion.net/license; you may not use this file except in compliance with the
Elastic License 2.0. #}
{% import_yaml 'curator/defaults.yaml' as CURATORDEFAULTS %}
{% set CURATORMERGED = salt['pillar.get']('curator', CURATORDEFAULTS.curator, merge=true) %}

39
salt/curator/soc_curator.yaml
View File

@@ -1,39 +0,0 @@
curator:
enabled:
description: You can enable or disable Curator.
helpLink: curator.html
elasticsearch:
index_settings:
so-beats:
delete: &delete
description: Age, in days, when Curator deletes the index.
helpLink: curator.html
forcedType: int
so-elasticsearch:
delete: *delete
so-firewall:
delete: *delete
so-ids:
delete: *delete
so-import:
delete: *delete
so-kratos:
delete: *delete
so-kibana:
delete: *delete
so-logstash:
delete: *delete
so-netflow:
delete: *delete
so-osquery:
delete: *delete
so-ossec:
delete: *delete
so-redis:
delete: *delete
so-strelka:
delete: *delete
so-syslog:
delete: *delete
so-zeek:
delete: *delete

21
salt/curator/sostatus.sls
View File

@@ -1,21 +0,0 @@
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{% from 'allowed_states.map.jinja' import allowed_states %}
{% if sls.split('.')[0] in allowed_states %}
append_so-curator_so-status.conf:
file.append:
- name: /opt/so/conf/so-status/so-status.conf
- text: so-curator
- unless: grep -q so-curator /opt/so/conf/so-status/so-status.conf
{% else %}
{{sls}}_state_not_allowed:
test.fail_without_changes:
- name: {{sls}}_state_not_allowed
{% endif %}

17
salt/curator/tools/sbin/so-curator-delete
View File

@@ -1,17 +0,0 @@
#!/bin/bash
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
APP=delete
lf=/tmp/$APP-pidLockFile
# create empty lock file if none exists
cat /dev/null >> $lf
read lastPID < $lf
# if lastPID is not null and a process with that pid exists , exit
[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit
echo $$ > $lf
docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/delete.yml > /dev/null 2>&1

12
salt/curator/tools/sbin/so-curator-restart
View File

@@ -1,12 +0,0 @@
#!/bin/bash
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
. /usr/sbin/so-common
/usr/sbin/so-restart curator $1

12
salt/curator/tools/sbin/so-curator-start
View File

@@ -1,12 +0,0 @@
#!/bin/bash
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
. /usr/sbin/so-common
/usr/sbin/so-start curator $1

12
salt/curator/tools/sbin/so-curator-stop
View File

@@ -1,12 +0,0 @@
#!/bin/bash
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
. /usr/sbin/so-common
/usr/sbin/so-stop curator $1

5
salt/docker/defaults.yaml
View File

@@ -159,11 +159,6 @@ docker:
custom_bind_mounts: []
extra_hosts: []
extra_env: []
'so-curator':
final_octet: 43
custom_bind_mounts: []
extra_hosts: []
extra_env: []
'so-elastic-fleet-package-registry':
final_octet: 44
port_bindings:

1
salt/docker/soc_docker.yaml
View File

@@ -60,7 +60,6 @@ docker:
so-strelka-gatekeeper: *dockerOptions
so-strelka-coordinator: *dockerOptions
so-elastalert: *dockerOptions
so-curator: *dockerOptions
so-elastic-fleet-package-registry: *dockerOptions
so-idh: *dockerOptions
so-elastic-agent: *dockerOptions

4
salt/elasticfleet/defaults.yaml
View File

@@ -76,6 +76,7 @@ elasticfleet:
- pulse_connect_secure
- redis
- sentinel_one
- snort
- snyk
- sonicwall_firewall
- sophos
@@ -85,9 +86,12 @@ elasticfleet:
- tcp
- tenable_sc
- ti_abusech
- ti_anomali
- ti_cybersixgill
- ti_misp
- ti_otx
- ti_recordedfuture
- ti_threatq
- udp
- vsphere
- windows

3
salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/import-zeek-logs.json
View File

@@ -27,6 +27,7 @@
}
}
}
}
},
"force": true
}
{%- endraw -%}

3
salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/zeek-logs.json
View File

@@ -28,6 +28,7 @@
}
}
}
}
},
"force": true
}
{%- endraw -%}

3
salt/elasticfleet/files/integrations-optional/sublime_platform.json
View File

@@ -39,6 +39,7 @@
}
}
}
}
},
"force": true
}
{%- endraw -%}

3
salt/elasticfleet/files/integrations/endpoints-initial/osquery.json
View File

@@ -16,5 +16,6 @@
}
}
}
}
},
"force": true
}

3
salt/elasticfleet/files/integrations/endpoints-initial/system-endpoints.json
View File

@@ -73,5 +73,6 @@
"system-system/metrics": {
"enabled": false
}
}
},
"force": true
}

3
salt/elasticfleet/files/integrations/endpoints-initial/windows-endpoints.json
View File

@@ -63,5 +63,6 @@
}
}
}
}
},
"force": true
}

3
salt/elasticfleet/files/integrations/grid-nodes_general/elasticsearch-logs.json
View File

@@ -102,5 +102,6 @@
}
}
}
}
},
"force": true
}

3
salt/elasticfleet/files/integrations/grid-nodes_general/idh-logs.json
View File

@@ -25,5 +25,6 @@
}
}
}
}
},
"force": true
}

3
salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json
View File

@@ -28,5 +28,6 @@
}
}
}
}
},
"force": true
}

3
salt/elasticfleet/files/integrations/grid-nodes_general/import-suricata-logs.json
View File

@@ -25,5 +25,6 @@
}
}
}
}
},
"force": true
}

3
salt/elasticfleet/files/integrations/grid-nodes_general/kratos-logs.json
View File

@@ -25,5 +25,6 @@
}
}
}
}
},
"force": true
}

3
salt/elasticfleet/files/integrations/grid-nodes_general/osquery-grid-nodes.json
View File

@@ -16,5 +16,6 @@
}
}
}
}
},
"force": true
}

3
salt/elasticfleet/files/integrations/grid-nodes_general/redis-logs.json
View File

@@ -72,5 +72,6 @@
}
}
}
}
},
"force": true
}

3
salt/elasticfleet/files/integrations/grid-nodes_general/soc-auth-sync-logs.json
View File

@@ -25,5 +25,6 @@
}
}
}
}
},
"force": true
}

3
salt/elasticfleet/files/integrations/grid-nodes_general/soc-salt-relay-logs.json
View File

@@ -25,5 +25,6 @@
}
}
}
}
},
"force": true
}

3
salt/elasticfleet/files/integrations/grid-nodes_general/soc-sensoroni-logs.json
View File

@@ -25,5 +25,6 @@
}
}
}
}
},
"force": true
}

3
salt/elasticfleet/files/integrations/grid-nodes_general/soc-server-logs.json
View File

@@ -25,5 +25,6 @@
}
}
}
}
},
"force": true
}

3
salt/elasticfleet/files/integrations/grid-nodes_general/strelka-logs.json
View File

@@ -25,5 +25,6 @@
}
}
}
}
},
"force": true
}

3
salt/elasticfleet/files/integrations/grid-nodes_general/suricata-logs.json
View File

@@ -25,5 +25,6 @@
}
}
}
}
},
"force": true
}

3
salt/elasticfleet/files/integrations/grid-nodes_general/syslog-tcp-514.json
View File

@@ -28,5 +28,6 @@
}
}
}
}
},
"force": true
}

3
salt/elasticfleet/files/integrations/grid-nodes_general/syslog-udp-514.json
View File

@@ -29,5 +29,6 @@
}
}
}
}
},
"force": true
}

3
salt/elasticfleet/files/integrations/grid-nodes_general/system-grid-nodes.json
View File

@@ -36,5 +36,6 @@
"system-system/metrics": {
"enabled": false
}
}
},
"force": true
}

3
salt/elasticfleet/files/integrations/grid-nodes_heavy/osquery-grid-nodes.json
View File

@@ -16,5 +16,6 @@
}
}
}
}
},
"force": true
}

3
salt/elasticfleet/files/integrations/grid-nodes_heavy/system-grid-nodes.json
View File

@@ -36,5 +36,6 @@
"system-system/metrics": {
"enabled": false
}
}
},
"force": true
}

176
salt/elasticsearch/defaults.yaml
View File

@@ -6737,6 +6737,50 @@ elasticsearch:
set_priority:
priority: 50
min_age: 30d
so-logs-snort_x_log:
index_sorting: False
index_template:
index_patterns:
- "logs-snort.log-*"
template:
settings:
index:
lifecycle:
name: so-logs-snort.log-logs
number_of_replicas: 0
composed_of:
- "logs-snort.log@package"
- "logs-snort.log@custom"
- "so-fleet_globals-1"
- "so-fleet_agent_id_verification-1"
priority: 501
data_stream:
hidden: false
allow_custom_routing: false
policy:
phases:
cold:
actions:
set_priority:
priority: 0
min_age: 30d
delete:
actions:
delete: {}
min_age: 365d
hot:
actions:
rollover:
max_age: 30d
max_primary_shard_size: 50gb
set_priority:
priority: 100
min_age: 0ms
warm:
actions:
set_priority:
priority: 50
min_age: 30d
so-logs-snyk_x_audit:
index_sorting: false
index_template:
@@ -7622,6 +7666,94 @@ elasticsearch:
set_priority:
priority: 50
min_age: 30d
so-logs-ti_anomali_x_threatstream:
index_sorting: False
index_template:
index_patterns:
- "logs-ti_anomali.threatstream-*"
template:
settings:
index:
lifecycle:
name: so-logs-ti_anomali.threatstream-logs
number_of_replicas: 0
composed_of:
- "logs-ti_anomali.threatstream@package"
- "logs-ti_anomali.threatstream@custom"
- "so-fleet_globals-1"
- "so-fleet_agent_id_verification-1"
priority: 501
data_stream:
hidden: false
allow_custom_routing: false
policy:
phases:
cold:
actions:
set_priority:
priority: 0
min_age: 30d
delete:
actions:
delete: {}
min_age: 365d
hot:
actions:
rollover:
max_age: 30d
max_primary_shard_size: 50gb
set_priority:
priority: 100
min_age: 0ms
warm:
actions:
set_priority:
priority: 50
min_age: 30d
so-logs-ti_cybersixgill_x_threat:
index_sorting: False
index_template:
index_patterns:
- "logs-ti_cybersixgill.threat-*"
template:
settings:
index:
lifecycle:
name: so-logs-ti_cybersixgill.threat-logs
number_of_replicas: 0
composed_of:
- "logs-ti_cybersixgill.threat@package"
- "logs-ti_cybersixgill.threat@custom"
- "so-fleet_globals-1"
- "so-fleet_agent_id_verification-1"
priority: 501
data_stream:
hidden: false
allow_custom_routing: false
policy:
phases:
cold:
actions:
set_priority:
priority: 0
min_age: 30d
delete:
actions:
delete: {}
min_age: 365d
hot:
actions:
rollover:
max_age: 30d
max_primary_shard_size: 50gb
set_priority:
priority: 100
min_age: 0ms
warm:
actions:
set_priority:
priority: 50
min_age: 30d
so-logs-ti_misp_x_threat:
index_sorting: false
index_template:
@@ -7842,6 +7974,50 @@ elasticsearch:
set_priority:
priority: 50
min_age: 30d
so-logs-ti_threatq_x_threat:
index_sorting: False
index_template:
index_patterns:
- "logs-ti_threatq.threat-*"
template:
settings:
index:
lifecycle:
name: so-logs-ti_threatq.threat-logs
number_of_replicas: 0
composed_of:
- "logs-ti_threatq.threat@package"
- "logs-ti_threatq.threat@custom"
- "so-fleet_globals-1"
- "so-fleet_agent_id_verification-1"
priority: 501
data_stream:
hidden: false
allow_custom_routing: false
policy:
phases:
cold:
actions:
set_priority:
priority: 0
min_age: 30d
delete:
actions:
delete: {}
min_age: 365d
hot:
actions:
rollover:
max_age: 30d
max_primary_shard_size: 50gb
set_priority:
priority: 100
min_age: 0ms
warm:
actions:
set_priority:
priority: 50
min_age: 30d
so-logs-vsphere_x_log:
index_sorting: False
index_template:

4
salt/elasticsearch/disabled.sls
View File

@@ -18,6 +18,10 @@ so-elasticsearch_so-status.disabled:
- name: /opt/so/conf/so-status/so-status.conf
- regex: ^so-elasticsearch$
so-elasticsearch-indices-delete:
cron.absent:
- identifier: so-elasticsearch-indices-delete
{% else %}
{{sls}}_state_not_allowed:

27
salt/elasticsearch/enabled.sls
View File

@@ -26,7 +26,12 @@ so-elasticsearch:
- networks:
- sobridge:
- ipv4_address: {{ DOCKER.containers['so-elasticsearch'].ip }}
- extra_hosts: {{ LOGSTASH_NODES }}
- extra_hosts:
{% for node in LOGSTASH_NODES %}
{% for hostname, ip in node.items() %}
- {{hostname}}:{{ip}}
{% endfor %}
{% endfor %}
{% if DOCKER.containers['so-elasticsearch'].extra_hosts %}
{% for XTRAHOST in DOCKER.containers['so-elasticsearch'].extra_hosts %}
- {{ XTRAHOST }}
@@ -195,6 +200,26 @@ so-elasticsearch-roles-load:
- require:
- docker_container: so-elasticsearch
- file: elasticsearch_sbin_jinja
{% if grains.role in ['so-eval', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-manager'] %}
so-curator-cluster-close:
cron.absent:
- identifier: so-curator-cluster-close
so-curator-cluster-delete:
cron.absent:
- identifier: so-curator-cluster-delete
so-elasticsearch-indices-delete:
cron.present:
- name: /usr/sbin/so-elasticsearch-indices-delete > /opt/so/log/elasticsearch/cron-elasticsearch-indices-delete.log 2>&1
- identifier: so-elasticsearch-indices-delete
- user: root
- minute: '*/5'
- hour: '*'
- daymonth: '*'
- month: '*'
- dayweek: '*'
{% endif %}
{% endif %}
{% else %}

4
salt/elasticsearch/soc_elasticsearch.yaml
View File

@@ -407,16 +407,20 @@ elasticsearch:
so-logs-sentinel_one_x_group: *indexSettings
so-logs-sentinel_one_x_threat: *indexSettings
so-logs-sonicwall_firewall_x_log: *indexSettings
so-logs-snort_x_log: *indexSettings
so-logs-symantec_endpoint_x_log: *indexSettings
so-logs-ti_abusech_x_malware: *indexSettings
so-logs-ti_abusech_x_malwarebazaar: *indexSettings
so-logs-ti_abusech_x_threatfox: *indexSettings
so-logs-ti_abusech_x_url: *indexSettings
so-logs-ti_anomali_x_threatstream: *indexSettings
so-logs-ti_cybersixgill_x_threat: *indexSettings
so-logs-ti_misp_x_threat: *indexSettings
so-logs-ti_misp_x_threat_attributes: *indexSettings
so-logs-ti_otx_x_threat: *indexSettings
so-logs-ti_recordedfuture_x_latest_ioc-template: *indexSettings
so-logs-ti_recordedfuture_x_threat: *indexSettings
so-logs-ti_threatq_x_threat: *indexSettings
so-logs-zscaler_zia_x_alerts: *indexSettings
so-logs-zscaler_zia_x_dns: *indexSettings
so-logs-zscaler_zia_x_firewall: *indexSettings

2
salt/elasticsearch/tools/sbin/so-elastic-clear
View File

@@ -67,7 +67,7 @@ if [ $SKIP -ne 1 ]; then
echo
echo "This script will delete all data (documents, indices, etc.) in the Elasticsearch database."
echo
echo "If you would like to proceed, please type "AGREE" and hit ENTER."
echo "If you would like to proceed, then type AGREE and press ENTER."
echo
# Read user input
read INPUT

2
salt/curator/tools/sbin/so-curator-cluster-delete → salt/elasticsearch/tools/sbin/so-elasticsearch-indices-delete
View File

@@ -14,4 +14,4 @@ read lastPID < $lf
[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit
echo $$ > $lf
/usr/sbin/so-curator-cluster-delete-delete
/usr/sbin/so-elasticsearch-indices-delete-delete

9
salt/curator/tools/sbin_jinja/so-curator-cluster-delete-delete → salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-indices-delete-delete
View File

@@ -9,8 +9,8 @@
{%- set ELASTICSEARCH_HOST = GLOBALS.node_ip -%}
{%- set RETENTION = salt['pillar.get']('elasticsearch:retention', ELASTICDEFAULTS.elasticsearch.retention, merge=true) -%}
LOG="/opt/so/log/curator/so-curator-cluster-delete.log"
ALERT_LOG="/opt/so/log/curator/alert.log"
LOG="/opt/so/log/elasticsearch/so-elasticsearch-indices-delete.log"
ALERT_LOG="/opt/so/log/elasticsearch/indices-delete-alert.log"
LOG_SIZE_LIMIT_GB=$(/usr/sbin/so-elasticsearch-cluster-space-total {{ RETENTION.retention_pct}})
LOG_SIZE_LIMIT=$(( "$LOG_SIZE_LIMIT_GB" * 1000 * 1000 * 1000 ))
ITERATION=0
@@ -51,8 +51,11 @@ while overlimit && [[ $ITERATION -lt $MAX_ITERATIONS ]]; do
if [ "${INDEX}" != "${CURRENT_WRITE_INDEX}" ]; then
# This should not be a write index, so we should be allowed to delete it
printf "\n$(date) - Used disk space exceeds LOG_SIZE_LIMIT (${LOG_SIZE_LIMIT_GB} GB) - Deleting ${INDEX} index...\n" >> ${LOG}
/usr/sbin/so-elasticsearch-query ${INDEX} -XDELETE >> ${LOG} 2>&1
/usr/sbin/so-elasticsearch-query ${INDEX} -XDELETE >> ${LOG} 2>&1
fi
else
printf "\n$(date) - Used disk space exceeds LOG_SIZE_LIMIT (${LOG_SIZE_LIMIT_GB} GB) - There is only one backing index (${INDEX}). Deleting ${DATASTREAM} data stream...\n" >> ${LOG}
/usr/sbin/so-elasticsearch-query _data_stream/$DATASTREAM -XDELETE >> ${LOG} 2>&1
fi
if ! overlimit ; then
exit

3
salt/firewall/containers.map.jinja
View File

@@ -2,7 +2,6 @@
{% if GLOBALS.role == 'so-eval' %}
{% set NODE_CONTAINERS = [
'so-curator',
'so-dockerregistry',
'so-elasticsearch',
'so-elastic-fleet',
@@ -25,7 +24,6 @@
{% elif GLOBALS.role == 'so-manager' or GLOBALS.role == 'so-standalone' or GLOBALS.role == 'so-managersearch' %}
{% set NODE_CONTAINERS = [
'so-curator',
'so-dockerregistry',
'so-elasticsearch',
'so-elastic-fleet',
@@ -56,7 +54,6 @@
{% elif GLOBALS.role == 'so-heavynode' %}
{% set NODE_CONTAINERS = [
'so-curator',
'so-elasticsearch',
'so-elastic-agent',
'so-logstash',

2
salt/logrotate/defaults.yaml
View File

@@ -80,7 +80,7 @@ logrotate:
- extension .log
- dateext
- dateyesterday
/opt/so/log/curator/*_x_log:
/opt/so/log/elasticsearch/*indices-delete*_x_log:
- daily
- rotate 14
- missingok

14
salt/logrotate/soc_logrotate.yaml
View File

@@ -56,13 +56,6 @@ logrotate:
multiline: True
global: True
forcedType: "[]string"
"/opt/so/log/curator/*_x_log":
description: List of logrotate options for this file.
title: /opt/so/log/curator/*.log
advanced: True
multiline: True
global: True
forcedType: "[]string"
"/opt/so/log/elasticfleet/*_x_log":
description: List of logrotate options for this file.
title: /opt/so/log/elasticfleet/*.log
@@ -77,6 +70,13 @@ logrotate:
multiline: True
global: True
forcedType: "[]string"
"/opt/so/log/elasticsearch/*indices-delete*_x_log":
description: List of logrotate options for this file.
title: /opt/so/log/elasticsearch/*indices-delete.log
advanced: True
multiline: True
global: True
forcedType: "[]string"
"/opt/so/log/suricata/*_x_log":
description: List of logrotate options for this file.
title: /opt/so/log/suricata/*.log

2
salt/manager/tools/sbin/so-elastic-auth-password-reset
View File

@@ -97,8 +97,6 @@ function soUserSync() {
salt -C 'G@role:so-standalone or G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managersearch or G@role:so-searchnode or G@role:so-heavynode' state.apply logstash queue=True
printf "\nApplying kibana state to the appropriate nodes.\n\n"
salt -C 'G@role:so-standalone or G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managersearch' state.apply kibana queue=True
printf "\nApplying curator state to the appropriate nodes.\n\n"
salt -C 'G@role:so-standalone or G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managersearch or G@role:so-searchnode or G@role:so-heavynode' state.apply curator queue=True
}
function highstateManager() {

14
salt/manager/tools/sbin/so-minion
View File

@@ -302,13 +302,6 @@ function add_strelka_to_minion() {
" " >> $PILLARFILE
}
function add_curator_to_minion() {
printf '%s\n'\
"curator:"\
" enabled: True"\
" " >> $PILLARFILE
}
function add_telegraf_to_minion() {
printf '%s\n'\
"telegraf:"\
@@ -435,7 +428,6 @@ function createEVAL() {
add_playbook_to_minion
add_elastalert_to_minion
add_kibana_to_minion
add_curator_to_minion
add_telegraf_to_minion
add_influxdb_to_minion
add_nginx_to_minion
@@ -458,7 +450,6 @@ function createSTANDALONE() {
add_elastalert_to_minion
add_kibana_to_minion
add_redis_to_minion
add_curator_to_minion
add_telegraf_to_minion
add_influxdb_to_minion
add_nginx_to_minion
@@ -478,7 +469,6 @@ function createMANAGER() {
add_elastalert_to_minion
add_kibana_to_minion
add_redis_to_minion
add_curator_to_minion
add_telegraf_to_minion
add_influxdb_to_minion
add_nginx_to_minion
@@ -498,7 +488,6 @@ function createMANAGERSEARCH() {
add_elastalert_to_minion
add_kibana_to_minion
add_redis_to_minion
add_curator_to_minion
add_telegraf_to_minion
add_influxdb_to_minion
add_nginx_to_minion
@@ -548,7 +537,6 @@ function createHEAVYNODE() {
add_sensor_to_minion
add_strelka_to_minion
add_redis_to_minion
add_curator_to_minion
add_telegraf_to_minion
}
@@ -679,4 +667,4 @@ case "$OPERATION" in
*)
usage
;;
esac
esac

9
salt/manager/tools/sbin/soup
View File

@@ -465,9 +465,12 @@ post_to_2.4.30() {
}
post_to_2.4.40() {
echo "Removing Curator close and delete files"
rm -f /opt/so/conf/curator/action/*-close.y*ml
rm -f /opt/so/conf/curator/action/logs*-delete.y*ml
echo "Stopping Curator"
so-curator-stop
echo "Removing Curator configuration"
rm -rf /opt/so/conf/curator/
rm -f /usr/sbin/so-curator-*
sed -i '/so-curator/d' /opt/so/conf/so-status/so-status.conf
}
repo_sync() {

4
salt/manager/tools/sbin_jinja/so-elastic-fleet-reset
View File

@@ -18,7 +18,7 @@ echo "This includes data previously ingested with Fleet such as Zeek and Suricat
echo "Deployed Elastic Agents will no longer be enrolled and will need to be reinstalled."
echo "This script should only be used as a last resort to reinstall Elastic Fleet."
echo
echo "If you would like to proceed, type AGREE and hit ENTER."
echo "If you would like to proceed, then type AGREE and press ENTER."
echo
# Read user input
read INPUT
@@ -93,4 +93,4 @@ so-elastic-fleet-setup
status "Re-installing Elastic Agent on all Grid Nodes..."
salt \* state.apply elasticfleet.install_agent_grid queue=True
status "Elastic Fleet Reset complete...."
status "Elastic Fleet Reset complete...."

16
salt/soc/defaults.yaml
View File

@@ -470,6 +470,18 @@ soc:
- rule.action
- rule.reason
- network.community_id
':pfsense:':
- soc_timestamp
- source.ip
- source.port
- destination.ip
- destination.port
- network.transport
- network.direction
- observer.ingress.interface.name
- event.action
- event.reason
- network.community_id
':osquery:':
- soc_timestamp
- source.ip
@@ -1348,7 +1360,7 @@ soc:
showSubtitle: true
- name: Firewall
description: Firewall events grouped by action
query: 'tags:firewall | groupby rule.action'
query: 'observer.type:firewall | groupby event.action'
showSubtitle: true
dashboards:
advanced: true
@@ -1551,7 +1563,7 @@ soc:
query: 'tags:s7* | groupby -sankey event.dataset source.ip destination.ip | groupby event.dataset | groupby source.ip | groupby destination.ip | groupby destination.port'
- name: Firewall
description: Firewall logs
query: 'tags:firewall | groupby -sankey rule.action interface.name | groupby rule.action | groupby interface.name | groupby network.transport | groupby source.ip | groupby destination.ip | groupby destination.port'
query: 'observer.type:firewall | groupby -sankey event.action observer.ingress.interface.name | groupby event.action | groupby observer.ingress.interface.name | groupby network.type | groupby network.transport | groupby source.ip | groupby destination.ip | groupby destination.port'
- name: VLAN
description: VLAN (Virtual Local Area Network) tagged logs
query: '* AND _exists_:network.vlan.id | groupby network.vlan.id | groupby source.ip | groupby -sankey source.ip destination.ip | groupby destination.ip | groupby destination.port | groupby event.dataset | groupby event.module | groupby observer.name | groupby source.geo.country_name | groupby destination.geo.country_name'

23
salt/soc/enabled.sls
View File

@@ -33,21 +33,26 @@ so-soc:
- /opt/so/conf/soc/soc_users_roles:/opt/sensoroni/rbac/users_roles:rw
- /opt/so/conf/soc/queue:/opt/sensoroni/queue:rw
- /opt/so/saltstack:/opt/so/saltstack:rw
- extra_hosts: {{ DOCKER_EXTRA_HOSTS }}
{% if DOCKER.containers['so-soc'].extra_hosts %}
{% for XTRAHOST in DOCKER.containers['so-soc'].extra_hosts %}
- extra_hosts:
{% for node in DOCKER_EXTRA_HOSTS %}
{% for hostname, ip in node.items() %}
- {{hostname}}:{{ip}}
{% endfor %}
{% endfor %}
{% if DOCKER.containers['so-soc'].extra_hosts %}
{% for XTRAHOST in DOCKER.containers['so-soc'].extra_hosts %}
- {{ XTRAHOST }}
{% endfor %}
{% endif %}
{% endfor %}
{% endif %}
- port_bindings:
{% for BINDING in DOCKER.containers['so-soc'].port_bindings %}
{% for BINDING in DOCKER.containers['so-soc'].port_bindings %}
- {{ BINDING }}
{% endfor %}
{% endfor %}
{% if DOCKER.containers['so-soc'].extra_env %}
- environment:
{% for XTRAENV in DOCKER.containers['so-soc'].extra_env %}
{% for XTRAENV in DOCKER.containers['so-soc'].extra_env %}
- {{ XTRAENV }}
{% endfor %}
{% endfor %}
{% endif %}
- watch:
- file: /opt/so/conf/soc/*

1
salt/telegraf/defaults.yaml
View File

@@ -48,6 +48,7 @@ telegraf:
- redis.sh
- sostatus.sh
import:
- influxdbsize.sh
- os.sh
- sostatus.sh
sensor:

2
salt/telegraf/etc/telegraf.conf
View File

@@ -186,7 +186,7 @@
#
# # Read stats from one or more Elasticsearch servers or clusters
{%- if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone', 'so-heavynode'] %}
{%- if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-import'] %}
[[inputs.elasticsearch]]
servers = ["https://{{ NODEIP }}:9200"]
cluster_stats = true

5
salt/top.sls
View File

@@ -74,7 +74,6 @@ base:
- suricata
- zeek
- strelka
- curator
- elastalert
- utility
- soctopus
@@ -104,7 +103,6 @@ base:
- redis
- elastic-fleet-package-registry
- kibana
- curator
- elastalert
- utility
- soctopus
@@ -140,7 +138,6 @@ base:
- suricata
- zeek
- strelka
- curator
- elastalert
- utility
- soctopus
@@ -179,7 +176,6 @@ base:
- elasticsearch
- logstash
- redis
- curator
- elastic-fleet-package-registry
- kibana
- elastalert
@@ -199,7 +195,6 @@ base:
- elasticsearch
- logstash
- redis
- curator
- strelka
- pcap
- suricata

3
setup/so-functions
View File

@@ -1412,7 +1412,7 @@ make_some_dirs() {
mkdir -p $local_salt_dir/salt/firewall/portgroups
mkdir -p $local_salt_dir/salt/firewall/ports
for THEDIR in bpf pcap elasticsearch ntp firewall redis backup influxdb strelka sensoroni curator soc soctopus docker zeek suricata nginx telegraf logstash soc manager kratos idstools idh elastalert global;do
for THEDIR in bpf pcap elasticsearch ntp firewall redis backup influxdb strelka sensoroni soc soctopus docker zeek suricata nginx telegraf logstash soc manager kratos idstools idh elastalert global;do
mkdir -p $local_salt_dir/pillar/$THEDIR
touch $local_salt_dir/pillar/$THEDIR/adv_$THEDIR.sls
touch $local_salt_dir/pillar/$THEDIR/soc_$THEDIR.sls
@@ -1555,7 +1555,6 @@ reserve_group_ids() {
logCmd "groupadd -g 931 logstash"
logCmd "groupadd -g 932 kibana"
logCmd "groupadd -g 933 elastalert"
logCmd "groupadd -g 934 curator"
logCmd "groupadd -g 937 zeek"
logCmd "groupadd -g 940 suricata"
logCmd "groupadd -g 941 stenographer"

6
setup/so-variables
View File

@@ -112,12 +112,6 @@ export sensoroni_pillar_file
adv_sensoroni_pillar_file="$local_salt_dir/pillar/sensoroni/adv_sensoroni.sls"
export adv_sensoroni_pillar_file
curator_pillar_file="$local_salt_dir/pillar/curator/soc_curator.sls"
export curator_pillar_file
adv_curator_pillar_file="$local_salt_dir/pillar/curator/adv_curator.sls"
export adv_curator_pillar_file
soctopus_pillar_file="$local_salt_dir/pillar/soctopus/soc_soctopus.sls"
export soctopus_pillar_file

7
setup/so-verify
View File

@@ -33,13 +33,17 @@ log_has_errors() {
# Ignore Failed: 0 since that is the salt state output, and we detect state failures
# via Result: False already.
# This is ignored for Ubuntu
# This is ignored for Ubuntu:
# Failed to restart snapd.mounts-pre.target: Operation refused, unit snapd.mounts-pre.target
# may be requested by dependency only (it is configured to refuse manual start/stop).
# Command failed with exit code is output during retry loops.
# "remove failed" is caused by a warning generated by upgrade of libwbclient
# Exit code 100 failure is likely apt-get running in the background, we wait for it to unlock.
# Failed to deduce dest mapping appears to occur when a shard isn't yet ready. Temporary.
grep -E "FAILED|Failed|failed|ERROR|Result: False|Error is not recoverable" "$setup_log" | \
grep -vE "The Salt Master has cached the public key for this node" | \
@@ -58,6 +62,7 @@ log_has_errors() {
grep -vE "remove failed" | \
grep -vE "Failed to restart snapd" | \
grep -vE "Login Failed Details" | \
grep -vE "Failed to deduce dest mappings" | \
grep -vE "response from daemon: unauthorized" | \
grep -vE "Reading first line of patchfile" | \
grep -vE "Command failed with exit code" | \