Compare commits

..
1 Commits
Author SHA1 Message Date
Mike Reeves 3e5a934ff8 Merge pull request #16238 from Security-Onion-Solutions/hotfix/3.3.0
Hotfix/3.3.0
2026-09-11 16:38:51 -04:00
119 changed files with 373 additions and 4435 deletions

No files matched your search

-1
View File
@@ -13,7 +13,6 @@ body:
- 3.1.0
- 3.2.0
- 3.3.0
- 3.4.0
- Other (please provide detail below)
validations:
required: true
+1 -27
View File
@@ -6,10 +6,6 @@ on:
- "salt/sensoroni/files/analyzers/**"
- "salt/manager/tools/sbin/**"
- "salt/_beacons/**"
- "salt/elastalert/files/modules/**"
- "salt/telegraf/tools/sbin_jinja/**"
- "salt/telegraf/defaults.yaml"
- "salt/telegraf/soc_telegraf.yaml"
jobs:
build:
@@ -19,7 +15,7 @@ jobs:
fail-fast: false
matrix:
python-version: ["3.14"]
python-code-path: ["salt/sensoroni/files/analyzers", "salt/manager/tools/sbin", "salt/_beacons", "salt/elastalert/files/modules/so"]
python-code-path: ["salt/sensoroni/files/analyzers", "salt/manager/tools/sbin", "salt/_beacons"]
steps:
- uses: actions/checkout@v3
@@ -38,25 +34,3 @@ jobs:
- name: Test with pytest
run: |
PYTHONPATH=${{ matrix.python-code-path }} pytest ${{ matrix.python-code-path }} --cov=${{ matrix.python-code-path }} --doctest-modules --cov-report=term --cov-fail-under=100 --cov-config=pytest.ini
telegraf-collector:
# so-container-stats is a jinja template rather than an importable module, so it gets its
# own job: the test renders it the way salt does, then drives it with a faked docker engine
# and cgroup tree. No container runtime is needed.
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Set up Python
uses: actions/setup-python@v3
with:
python-version: "3.14"
- name: Install dependencies
run: |
python -m pip install --upgrade pip
python -m pip install flake8 pytest jinja2 pyyaml
- name: Lint with flake8
run: |
flake8 salt/telegraf/tools/sbin_jinja/so-container-stats_test.py --config=pytest.ini
- name: Test with pytest
run: |
pytest salt/telegraf/tools/sbin_jinja/so-container-stats_test.py -v
+1 -1
View File
@@ -1 +1 @@
20260911
+1 -1
View File
@@ -1 +1 @@
3.4.0
3.3.0
+5 -20
View File
@@ -117,25 +117,14 @@ elastic_curl_config:
{% endif %}
# A non-root owner here can chmod the directory and replace any script in it, including
# the root-owned ones. 555 is the mode the filesystem RPM ships; root ignores it anyway.
usr_sbin_perms:
file.directory:
- name: /usr/sbin
- user: root
- group: root
- mode: 555
common_sbin:
file.recurse:
- name: /usr/sbin
- source: salt://common/tools/sbin
- user: root
- group: root
- user: 939
- group: 939
- file_mode: 755
- show_changes: False
- require:
- file: usr_sbin_perms
{% if GLOBALS.role == 'so-heavynode' %}
- exclude_pat:
- so-pcap-import
@@ -170,8 +159,8 @@ common_sbin_jinja:
file.recurse:
- name: /usr/sbin
- source: salt://common/tools/sbin_jinja
- user: root
- group: root
- user: 939
- group: 939
- file_mode: 755
- template: jinja
- show_changes: False
@@ -184,8 +173,6 @@ so-status_script:
file.managed:
- name: /usr/sbin/so-status
- source: salt://common/tools/sbin/so-status
- user: root
- group: root
- mode: 755
{% if GLOBALS.is_sensor %}
@@ -217,11 +204,9 @@ sostatus_log:
- replace: False
# Install sostatus check cron. This is used to populate Grid.
# telegraf reads status.log on the same minute boundary this runs, so write aside and rename
# rather than truncating the file it is reading
so-status_check_cron:
cron.present:
- name: '/usr/sbin/so-status -j > /opt/so/log/sostatus/status.log.tmp 2>&1; mv -f /opt/so/log/sostatus/status.log.tmp /opt/so/log/sostatus/status.log'
- name: '/usr/sbin/so-status -j > /opt/so/log/sostatus/status.log 2>&1'
- identifier: so-status_check_cron
- user: root
- minute: '*/1'
+14 -42
View File
@@ -18,61 +18,47 @@ copy_so-common_common_tools_sbin:
- name: /opt/so/saltstack/default/salt/common/tools/sbin/so-common
- source: {{UPDATE_DIR}}/salt/common/tools/sbin/so-common
- force: True
- user: root
- group: root
- mode: 755
- preserve: True
copy_so-image-common_common_tools_sbin:
file.copy:
- name: /opt/so/saltstack/default/salt/common/tools/sbin/so-image-common
- source: {{UPDATE_DIR}}/salt/common/tools/sbin/so-image-common
- force: True
- user: root
- group: root
- mode: 755
- preserve: True
copy_soup_manager_tools_sbin:
file.copy:
- name: /opt/so/saltstack/default/salt/manager/tools/sbin/soup
- source: {{UPDATE_DIR}}/salt/manager/tools/sbin/soup
- force: True
- user: root
- group: root
- mode: 755
- preserve: True
copy_so-firewall_manager_tools_sbin:
file.copy:
- name: /opt/so/saltstack/default/salt/manager/tools/sbin/so-firewall
- source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-firewall
- force: True
- user: root
- group: root
- mode: 755
- preserve: True
copy_so-yaml_manager_tools_sbin:
file.copy:
- name: /opt/so/saltstack/default/salt/manager/tools/sbin/so-yaml.py
- source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-yaml.py
- force: True
- user: root
- group: root
- mode: 755
- preserve: True
copy_so-repo-sync_manager_tools_sbin:
file.copy:
- name: /opt/so/saltstack/default/salt/manager/tools/sbin/so-repo-sync
- source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-repo-sync
- user: root
- group: root
- mode: 755
- preserve: True
copy_bootstrap-salt_manager_tools_sbin:
file.copy:
- name: /opt/so/saltstack/default/salt/salt/scripts/bootstrap-salt.sh
- source: {{UPDATE_DIR}}/salt/salt/scripts/bootstrap-salt.sh
- user: root
- group: root
- mode: 644
- preserve: True
# This section is used to put the new script in place so that it can be called during soup.
# It is faster than calling the states that normally manage them to put them in place.
@@ -81,60 +67,46 @@ copy_so-common_sbin:
- name: /usr/sbin/so-common
- source: {{UPDATE_DIR}}/salt/common/tools/sbin/so-common
- force: True
- user: root
- group: root
- mode: 755
- preserve: True
copy_so-image-common_sbin:
file.copy:
- name: /usr/sbin/so-image-common
- source: {{UPDATE_DIR}}/salt/common/tools/sbin/so-image-common
- force: True
- user: root
- group: root
- mode: 755
- preserve: True
copy_soup_sbin:
file.copy:
- name: /usr/sbin/soup
- source: {{UPDATE_DIR}}/salt/manager/tools/sbin/soup
- force: True
- user: root
- group: root
- mode: 755
- preserve: True
copy_so-firewall_sbin:
file.copy:
- name: /usr/sbin/so-firewall
- source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-firewall
- force: True
- user: root
- group: root
- mode: 755
- preserve: True
copy_so-yaml_sbin:
file.copy:
- name: /usr/sbin/so-yaml.py
- source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-yaml.py
- force: True
- user: root
- group: root
- mode: 755
- preserve: True
copy_so-repo-sync_sbin:
file.copy:
- name: /usr/sbin/so-repo-sync
- source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-repo-sync
- force: True
- user: root
- group: root
- mode: 755
- preserve: True
copy_bootstrap-salt_sbin:
file.copy:
- name: /usr/sbin/bootstrap-salt.sh
- source: {{UPDATE_DIR}}/salt/salt/scripts/bootstrap-salt.sh
- force: True
- user: root
- group: root
- mode: 755
- preserve: True
+1 -2
View File
@@ -240,8 +240,7 @@ copy_new_files() {
cd $UPDATE_DIR
rsync -a salt $DEFAULT_SALT_DIR/ --delete "${EXCLUDE_ARGS[@]}"
rsync -a pillar $DEFAULT_SALT_DIR/ --delete "${EXCLUDE_ARGS[@]}"
# Root-executed code; SOC only needs to read it. Local dirs stay socore-owned.
chown -R root:root $DEFAULT_SALT_DIR/
chown -R socore:socore $DEFAULT_SALT_DIR/
cd /tmp
}
+6 -19
View File
@@ -9,7 +9,6 @@ import sys
import subprocess
import os
import json
import tempfile
sys.path.append('/opt/saltstack/salt/lib/python3.10/site-packages/')
import salt.config
@@ -18,21 +17,6 @@ import salt.loader
__opts__ = salt.config.minion_config('/etc/salt/minion')
__grains__ = salt.loader.grains(__opts__)
def write_atomic(path, value):
# telegraf reads these files on its own schedule; replacing them by rename means it never
# reads a truncated file and reports an empty value as if it were real
directory = os.path.dirname(path)
handle, temp = tempfile.mkstemp(dir=directory)
try:
with os.fdopen(handle, 'w') as f:
f.write(str(value))
os.chmod(temp, 0o644)
os.replace(temp, path)
except Exception:
os.path.exists(temp) and os.unlink(temp)
raise
def check_needs_restarted():
osfam = __grains__['os_family']
val = '0'
@@ -50,7 +34,8 @@ def check_needs_restarted():
else:
fail("Unsupported OS")
write_atomic(outfile, val)
with open(outfile, 'w') as f:
f.write(val)
def check_for_fps():
feat = 'fps'
@@ -71,7 +56,8 @@ def check_for_fps():
# Unknown, so assume 0
fps = 0
write_atomic('/opt/so/log/sostatus/fps_enabled', fps)
with open('/opt/so/log/sostatus/fps_enabled', 'w') as f:
f.write(str(fps))
def check_for_lks():
feat = 'Lks'
@@ -94,7 +80,8 @@ def check_for_lks():
lks = 1
if lks:
break
write_atomic('/opt/so/log/sostatus/lks_enabled', lks)
with open('/opt/so/log/sostatus/lks_enabled', 'w') as f:
f.write(str(lks))
def fail(msg):
print(msg, file=sys.stderr)
+1 -3
View File
@@ -177,8 +177,6 @@ if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Unexpected authorization header" # expected WARN log lines indicating invalid auth header
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Missing ory_kratos_session cookie" # expected WARN log lines indicating invalid auth header
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Static assets preprocessor only supports GET and HEAD requests" # expected WARN log lines indicating invalid auth header
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|respondError" # respondError is a function name, output via http middleware as standard request logging
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|GET /kibana/" # Ignore Kibana queries with triggered words
fi
if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
@@ -242,7 +240,7 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|marked for removal" # docker container getting recycled
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|tcp 127.0.0.1:6791: bind: address already in use" # so-elastic-fleet agent restarting. Seen starting w/ 8.18.8 https://github.com/elastic/kibana/issues/201459
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|TransformTask\] \[logs-.*user so_kibana lacks the required permissions" # Known issue with integrations starting transform jobs that are explicitly not allowed to start as a system user
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|manifest unknown" # so-dockerregistry logs a tag lookup miss during image copy; not tied to one docker version
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|manifest unknown" # appears in so-dockerregistry log for so-tcpreplay following docker upgrade to 29.2.1-1
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Could not index event to Elasticsearch.*\"version\" => \"9.0.8\"" # Expected during Elastic upgrade temporarily, as policies referencing older pipelines are updated
fi
+47 -78
View File
@@ -8,37 +8,21 @@
# Elastic License 2.0.
SENSOR_DIR="${SENSOR_DIR:-/nsm}"
SENSOR_DIR='/nsm'
CRIT_DISK_USAGE=90
LOG="${LOG:-/opt/so/log/sensor_clean.log}"
LOCK="${LOCK:-/var/tmp/so-sensor-clean.lock}"
MAX_PASSES=100
CUR_USAGE=$(df -P $SENSOR_DIR | tail -1 | awk '{print $5}' | tr -d %)
LOG="/opt/so/log/sensor_clean.log"
TODAY=$(date -u "+%Y-%m-%d")
ZEEK_LOGS="$SENSOR_DIR/zeek/logs"
STRELKA_FILES="$SENSOR_DIR/strelka/processed"
SURICATA_LOGS="$SENSOR_DIR/suricata"
PCAPS="$SENSOR_DIR/pcapout"
log() {
echo "$(date) - $*" >>"$LOG"
}
disk_usage() {
df -P "$SENSOR_DIR" | tail -1 | awk '{print $5}' | tr -d %
}
disk_avail() {
df -P "$SENSOR_DIR" | tail -1 | awk '{print $4}'
}
# sets REMOVED=1 if anything was actually deleted
clean() {
## find the oldest Zeek logs directory
OLDEST_DIR=$(ls "$ZEEK_LOGS" 2>/dev/null | grep -v "current" | grep -v "stats" | grep -v "packetloss" | grep -v "zeek_clean" | sort | head -n 1)
if [ -n "$OLDEST_DIR" ]; then
log "Removing directory: $ZEEK_LOGS/$OLDEST_DIR"
rm -rf "$ZEEK_LOGS/$OLDEST_DIR"
REMOVED=1
OLDEST_DIR=$(ls /nsm/zeek/logs/ | grep -v "current" | grep -v "stats" | grep -v "packetloss" | grep -v "zeek_clean" | sort | head -n 1)
if [ -z "$OLDEST_DIR" -o "$OLDEST_DIR" == ".." -o "$OLDEST_DIR" == "." ]; then
echo "$(date) - No old Zeek logs available to clean up in /nsm/zeek/logs/" >>$LOG
#exit 0
else
echo "$(date) - Removing directory: /nsm/zeek/logs/$OLDEST_DIR" >>$LOG
rm -rf /nsm/zeek/logs/"$OLDEST_DIR"
fi
## Remarking for now, as we are moving extracted files to /nsm/strelka/processed
@@ -59,73 +43,58 @@ clean() {
#fi
## Clean up Zeek extracted files processed by Strelka
OLDEST_STRELKA=$(find "$STRELKA_FILES" -type f -printf '%T+ %p\n' 2>/dev/null | sort -n | head -n 1)
if [ -n "$OLDEST_STRELKA" ]; then
STRELKA_FILES='/nsm/strelka/processed'
OLDEST_STRELKA=$(find $STRELKA_FILES -type f -printf '%T+ %p\n' | sort -n | head -n 1)
if [ -z "$OLDEST_STRELKA" -o "$OLDEST_STRELKA" == ".." -o "$OLDEST_STRELKA" == "." ]; then
echo "$(date) - No old files available to clean up in $STRELKA_FILES" >>$LOG
else
OLDEST_STRELKA_DATE=$(echo $OLDEST_STRELKA | awk '{print $1}' | cut -d+ -f1)
log "Removing extracted files for $OLDEST_STRELKA_DATE"
REMOVED=1
find "$STRELKA_FILES" -type f -printf '%T+ %p\n' 2>/dev/null | grep $OLDEST_STRELKA_DATE | awk '{print $2}' | while read FILE; do
log "Removing file: $FILE"
OLDEST_STRELKA_FILE=$(echo $OLDEST_STRELKA | awk '{print $2}')
echo "$(date) - Removing extracted files for $OLDEST_STRELKA_DATE" >>$LOG
find $STRELKA_FILES -type f -printf '%T+ %p\n' | grep $OLDEST_STRELKA_DATE | awk '{print $2}' | while read FILE; do
echo "$(date) - Removing file: $FILE" >>$LOG
rm -f "$FILE"
done
fi
## Clean up Suricata log files
OLDEST_SURICATA=$(find "$SURICATA_LOGS" -type f -printf '%T+ %p\n' 2>/dev/null | sort -n | head -n 1)
if [ -n "$OLDEST_SURICATA" ]; then
SURICATA_LOGS='/nsm/suricata'
OLDEST_SURICATA=$(find $SURICATA_LOGS -type f -printf '%T+ %p\n' | sort -n | head -n 1)
if [[ -z "$OLDEST_SURICATA" ]] || [[ "$OLDEST_SURICATA" == ".." ]] || [[ "$OLDEST_SURICATA" == "." ]]; then
echo "$(date) - No old files available to clean up in $SURICATA_LOGS" >>$LOG
else
OLDEST_SURICATA_DATE=$(echo $OLDEST_SURICATA | awk '{print $1}' | cut -d+ -f1)
log "Removing logs for $OLDEST_SURICATA_DATE"
REMOVED=1
find "$SURICATA_LOGS" -type f -printf '%T+ %p\n' 2>/dev/null | grep $OLDEST_SURICATA_DATE | awk '{print $2}' | while read FILE; do
log "Removing file: $FILE"
OLDEST_SURICATA_FILE=$(echo $OLDEST_SURICATA | awk '{print $2}')
echo "$(date) - Removing logs for $OLDEST_SURICATA_DATE" >>$LOG
find $SURICATA_LOGS -type f -printf '%T+ %p\n' | grep $OLDEST_SURICATA_DATE | awk '{print $2}' | while read FILE; do
echo "$(date) - Removing file: $FILE" >>$LOG
rm -f "$FILE"
done
fi
## Clean up extracted pcaps
OLDEST_PCAP=$(find "$PCAPS" -type f -printf '%T+ %p\n' 2>/dev/null | sort -n | head -n 1)
if [ -n "$OLDEST_PCAP" ]; then
PCAPS='/nsm/pcapout'
OLDEST_PCAP=$(find $PCAPS -type f -printf '%T+ %p\n' | sort -n | head -n 1)
if [ -z "$OLDEST_PCAP" -o "$OLDEST_PCAP" == ".." -o "$OLDEST_PCAP" == "." ]; then
echo "$(date) - No old files available to clean up in $PCAPS" >>$LOG
else
OLDEST_PCAP_DATE=$(echo $OLDEST_PCAP | awk '{print $1}' | cut -d+ -f1)
log "Removing extracted files for $OLDEST_PCAP_DATE"
REMOVED=1
find "$PCAPS" -type f -printf '%T+ %p\n' 2>/dev/null | grep $OLDEST_PCAP_DATE | awk '{print $2}' | while read FILE; do
log "Removing file: $FILE"
OLDEST_PCAP_FILE=$(echo $OLDEST_PCAP | awk '{print $2}')
echo "$(date) - Removing extracted files for $OLDEST_PCAP_DATE" >>$LOG
find $PCAPS -type f -printf '%T+ %p\n' | grep $OLDEST_PCAP_DATE | awk '{print $2}' | while read FILE; do
echo "$(date) - Removing file: $FILE" >>$LOG
rm -f "$FILE"
done
fi
}
# Only one instance at a time; the lock is the fd, so it releases on any exit
exec 9>"$LOCK" || exit 1
if ! flock -n 9; then
log "another so-sensor-clean is already running (lock $LOCK held); exiting"
exit 0
# Check to see if we are already running
NUM_RUNNING=$(pgrep -cf "/bin/bash /usr/sbin/so-sensor-clean")
[ "$NUM_RUNNING" -gt 1 ] && echo "$(date) - $NUM_RUNNING sensor clean script processes running...exiting." >>$LOG && exit 0
if [ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ]; then
while [ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ]; do
clean
CUR_USAGE=$(df -P $SENSOR_DIR | tail -1 | awk '{print $5}' | tr -d %)
done
fi
CUR_USAGE=$(disk_usage)
[ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ] || exit 0
log "$SENSOR_DIR at ${CUR_USAGE}% (threshold ${CRIT_DISK_USAGE}%); starting cleanup"
PASS=0
while [ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ]; do
PASS=$((PASS + 1))
if [ "$PASS" -gt "$MAX_PASSES" ]; then
log "stopping after $MAX_PASSES passes; $SENSOR_DIR still at ${CUR_USAGE}%"
break
fi
REMOVED=0
BEFORE=$(disk_avail)
clean
CUR_USAGE=$(disk_usage)
if [ "$REMOVED" -eq 0 ]; then
log "nothing left to remove in $ZEEK_LOGS, $STRELKA_FILES, $SURICATA_LOGS, $PCAPS; $SENSOR_DIR still at ${CUR_USAGE}% - space is consumed outside of NSM cleanup scope"
break
fi
if [ "$(disk_avail)" -le "$BEFORE" ]; then
log "pass $PASS freed no space; $SENSOR_DIR still at ${CUR_USAGE}% - stopping until next run"
break
fi
done
+1 -3
View File
@@ -125,6 +125,4 @@ else
RAIDSTATUS=1
fi
# telegraf reads this file; write aside and rename so it never sees a half-written file
echo "nsmraid=$RAIDSTATUS" > /opt/so/log/raid/status.log.tmp
mv -f /opt/so/log/raid/status.log.tmp /opt/so/log/raid/status.log
echo "nsmraid=$RAIDSTATUS" > /opt/so/log/raid/status.log
+2 -9
View File
@@ -1,12 +1,6 @@
docker:
range: '172.17.1.0/24'
gateway: '172.17.1.1'
networks:
sobridge: {}
soauth:
range: '172.17.2.0/24'
gateway: '172.17.2.1'
manager_only: True
ulimits:
- name: nofile
soft: 1048576
@@ -64,18 +58,18 @@ docker:
ulimits: []
'so-kratos':
final_octet: 28
networks: ['soauth']
port_bindings:
- 0.0.0.0:4433:4433
- 0.0.0.0:4434:4434
custom_bind_mounts: []
extra_hosts: []
extra_env: []
ulimits: []
'so-hydra':
final_octet: 30
networks: ['soauth']
port_bindings:
- 0.0.0.0:4444:4444
- 0.0.0.0:4445:4445
custom_bind_mounts: []
extra_hosts: []
extra_env: []
@@ -134,7 +128,6 @@ docker:
ulimits: []
'so-soc':
final_octet: 34
networks: ['sobridge', 'soauth']
port_bindings:
- 0.0.0.0:9822:9822
custom_bind_mounts: []
+3 -21
View File
@@ -1,26 +1,8 @@
{% import_yaml 'docker/defaults.yaml' as DOCKERDEFAULTS %}
{% set DOCKERMERGED = salt['pillar.get']('docker', DOCKERDEFAULTS.docker, merge=True) %}
{% if DOCKERMERGED.networks.sobridge is not mapping %}
{% do DOCKERMERGED.networks.update({'sobridge': {}}) %}
{% endif %}
{% do DOCKERMERGED.networks['sobridge'].update({'range': DOCKERMERGED.range, 'gateway': DOCKERMERGED.gateway}) %}
{% for netname, net in DOCKERMERGED.networks.items() %}
{% set RANGESPLIT = net.range.split('.') %}
{% do net.update({'prefix': RANGESPLIT[0] ~ '.' ~ RANGESPLIT[1] ~ '.' ~ RANGESPLIT[2] ~ '.'}) %}
{% endfor %}
{% set RANGESPLIT = DOCKERMERGED.range.split('.') %}
{% set FIRSTTHREE = RANGESPLIT[0] ~ '.' ~ RANGESPLIT[1] ~ '.' ~ RANGESPLIT[2] ~ '.' %}
{% for container, vals in DOCKERMERGED.containers.items() %}
{% set CONTAINER_NETS = vals.get('networks', ['sobridge']) %}
{% set IPS = {} %}
{% for netname in CONTAINER_NETS %}
{% do IPS.update({netname: DOCKERMERGED.networks[netname].prefix ~ vals.final_octet}) %}
{% endfor %}
{% do DOCKERMERGED.containers[container].update({
'networks': CONTAINER_NETS,
'ips': IPS,
'network': CONTAINER_NETS[0],
'ip': IPS[CONTAINER_NETS[0]]
}) %}
{% do DOCKERMERGED.containers[container].update({'ip': FIRSTTHREE ~ DOCKERMERGED.containers[container].final_octet}) %}
{% endfor %}
+10 -14
View File
@@ -18,10 +18,10 @@ dockergroup:
dockerheldpackages:
pkg.installed:
- pkgs:
- containerd.io: 2.3.6-1.el9
- docker-ce: 3:29.8.1-1.el9
- docker-ce-cli: 1:29.8.1-1.el9
- docker-ce-rootless-extras: 29.8.1-1.el9
- containerd.io: 2.2.1-1.el9
- docker-ce: 3:29.2.1-1.el9
- docker-ce-cli: 1:29.2.1-1.el9
- docker-ce-rootless-extras: 29.2.1-1.el9
- hold: True
- update_holds: True
@@ -71,19 +71,15 @@ dockerreserveports:
- source: salt://common/files/99-reserved-ports.conf
- name: /etc/sysctl.d/99-reserved-ports.conf
{% for NETNAME, NETWORK in DOCKERMERGED.networks.items() %}
{% if not NETWORK.get('manager_only') or GLOBALS.get('is_manager', False) %}
sos_docker_net_{{ NETNAME }}:
sos_docker_net:
docker_network.present:
- name: {{ NETNAME }}
- subnet: {{ NETWORK.range }}
- gateway: {{ NETWORK.gateway }}
- name: sobridge
- subnet: {{ DOCKERMERGED.range }}
- gateway: {{ DOCKERMERGED.gateway }}
- options:
com.docker.network.bridge.name: '{{ NETNAME }}'
com.docker.network.bridge.name: 'sobridge'
com.docker.network.driver.mtu: '1500'
com.docker.network.bridge.enable_ip_masquerade: 'true'
com.docker.network.bridge.enable_icc: 'true'
com.docker.network.bridge.host_binding_ipv4: '0.0.0.0'
- unless: ip l | grep {{ NETNAME }}
{% endif %}
{% endfor %}
- unless: ip l | grep sobridge
-44
View File
@@ -7,40 +7,6 @@ docker:
description: Default docker IP range for containers.
helpLink: docker
advanced: True
networks:
sobridge:
description: |
The default docker network, carrying most containers. Its range and gateway are taken
from the docker.range and docker.gateway settings above rather than set here.
helpLink: docker
readonly: True
advanced: True
global: True
soauth:
range:
description: |
IP range for the soauth docker network, an isolated network for the authentication
services, so that the Kratos and Hydra admin APIs are only reachable from the
containers placed on it.
helpLink: docker
readonly: True
advanced: True
global: True
gateway:
description: Gateway for the soauth docker network.
helpLink: docker
readonly: True
advanced: True
global: True
manager_only:
description: |
Limits the soauth network to grid members running the authentication containers,
instead of creating it on every node.
helpLink: docker
readonly: True
advanced: True
global: True
forcedType: bool
ulimits:
description: |
Default ulimit settings applied to all containers via the Docker daemon. Each entry specifies a resource name (e.g. nofile, memlock, core, nproc) with soft and hard limits. Individual container ulimits override these defaults. Valid resource names include: cpu, fsize, data, stack, core, rss, nproc, nofile, memlock, as, locks, sigpending, msgqueue, nice, rtprio, rttime.
@@ -68,16 +34,6 @@ docker:
readonly: True
advanced: True
global: True
networks:
description: |
Docker networks this container is attached to. The first entry is the container's
primary network and determines the address its published ports are forwarded to.
Defaults to sobridge when unset.
helpLink: docker
readonly: True
advanced: True
global: True
forcedType: "[]string"
port_bindings:
description: List of port bindings for the container.
helpLink: docker
+2 -3
View File
@@ -21,7 +21,6 @@ elastalert:
- gid: 933
- home: /opt/so/conf/elastalert
- createhome: False
- shell: /sbin/nologin
elastalogdir:
file.directory:
@@ -34,8 +33,8 @@ elastalert_sbin:
file.recurse:
- name: /usr/sbin
- source: salt://elastalert/tools/sbin
- user: root
- group: root
- user: 933
- group: 939
- file_mode: 755
#elastalert_sbin_jinja:
+1 -1
View File
@@ -10,7 +10,7 @@ elastalert:
buffer_time:
minutes: 10
old_query_limit:
minutes: 1440
minutes: 5
es_port: 9200
es_conn_timeout: 55
max_query_size: 5000
@@ -1,80 +0,0 @@
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
from datetime import datetime
import json
import logging
import sys
import types
# stand-ins when ElastAlert isn't installed (CI)
try:
import elastalert.alerts # noqa: F401
except ImportError:
class Alerter:
def __init__(self, rule):
self.rule = rule
class DateTimeEncoder(json.JSONEncoder):
def default(self, obj):
return obj.isoformat() if hasattr(obj, 'isoformat') else json.JSONEncoder.default(self, obj)
class EAException(Exception):
pass
def lookup_es_key(doc, term):
for part in term.split('.'):
if not isinstance(doc, dict) or part not in doc:
return None
doc = doc[part]
return doc
def ts_to_dt(value):
return value if isinstance(value, datetime) else datetime.fromisoformat(value)
def elasticsearch_client(conf):
return None # tests set the alerter's client
alerts = types.ModuleType('elastalert.alerts')
alerts.Alerter = Alerter
alerts.DateTimeEncoder = DateTimeEncoder
util = types.ModuleType('elastalert.util')
util.EAException = EAException
util.elastalert_logger = logging.getLogger('elastalert')
util.lookup_es_key = lookup_es_key
util.ts_to_dt = ts_to_dt
util.elasticsearch_client = elasticsearch_client
package = types.ModuleType('elastalert')
package.alerts = alerts
package.util = util
sys.modules.update({'elastalert': package, 'elastalert.alerts': alerts, 'elastalert.util': util})
# stand-ins when elasticsearch-py isn't installed (CI)
try:
import elasticsearch.exceptions # noqa: F401
except ImportError:
class ElasticsearchException(Exception):
pass
class TransportError(ElasticsearchException):
pass
class ConnectionError(TransportError):
pass
class ConflictError(TransportError):
pass
class RequestError(TransportError):
pass
exceptions = types.ModuleType('elasticsearch.exceptions')
for cls in (ElasticsearchException, TransportError, ConnectionError, ConflictError, RequestError):
setattr(exceptions, cls.__name__, cls)
es_package = types.ModuleType('elasticsearch')
es_package.exceptions = exceptions
sys.modules.update({'elasticsearch': es_package, 'elasticsearch.exceptions': exceptions})
@@ -1,269 +1,63 @@
# -*- coding: utf-8 -*-
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
from datetime import datetime, timezone
import hashlib
import ipaddress
import json
import re
import uuid
from time import gmtime, strftime
import requests,json
from elastalert.alerts import Alerter
import urllib3
from elasticsearch.exceptions import ConflictError, ElasticsearchException, RequestError
from elastalert.alerts import Alerter, DateTimeEncoder
from elastalert.util import EAException, elastalert_logger, elasticsearch_client, lookup_es_key, ts_to_dt
# grid runs verify_certs: false; also quiets ElastAlert's own queries
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
ALERT_INDEX = 'logs-detections.alerts-so'
# ES error text kept in logs and alerts
ERROR_TEXT_LIMIT = 500
# a match missing backend columns (window_start, count, @timestamp); the alert is still written
MATCH_ERRORS = (KeyError, TypeError, ValueError)
class SecurityOnionESAlerter(Alerter):
"""
Use matched data to create alerts in Elasticsearch.
"""
required_options = {'detection_title', 'sigma_level'}
optional_fields = ['sigma_category', 'sigma_product', 'sigma_service', 'sigma_correlation']
# count column and default summary per type; stored alert data, so not localized
CORRELATION_COUNTS = {
'event_count': ('event_count', '%count% events'),
'value_count': ('value_count', '%count% distinct values'),
'temporal': ('event_type_count', '%count% correlated rules matched'),
'value_sum': ('value_sum', 'total %count%'),
'value_avg': ('value_avg', 'average %count%'),
'value_percentile': ('value_percentile', 'percentile %count%'),
'value_median': ('value_median', 'median %count%'),
}
PLACEHOLDER = re.compile(r'%([^%\s]+)%')
# group-by fields copied into ECS related.*
RELATED_USERS = {'user.name', 'winlog.event_data.TargetUserName', 'winlog.event_data.SubjectUserName'}
RELATED_HOSTS = {'host.name', 'host.hostname', 'winlog.computer_name'}
def __init__(self, rule):
super().__init__(rule)
# uses the grid's TLS, auth and timeout settings
self.es = elasticsearch_client(rule)
@property
def is_correlation(self):
return bool(self.rule.get('sigma_correlation'))
def query_keys(self):
"""compound_query_key holds the list; query_key is flattened to a string."""
if self.rule.get('compound_query_key'):
return self.rule['compound_query_key']
if self.rule.get('query_key'):
return [self.rule['query_key']]
return []
def alert_id(self, match):
"""Stable id: window end + group values for correlations, source _id otherwise; random without one."""
if self.is_correlation:
# ungrouped rows have a hashed _id that changes with the count
values = ''.join(f"|{lookup_es_key(match, k)}" for k in self.query_keys())
key = f"{self.rule['detection_public_id']}|{ts_to_dt(match['@timestamp']).isoformat()}{values}"
elif match.get('_id'):
key = f"{self.rule['detection_public_id']}|{match['_id']}"
else:
return uuid.uuid4().hex
return hashlib.sha256(key.encode('utf-8')).hexdigest()
def group(self, match):
"""Group-by values, joined like ElastAlert's realert key."""
return ', '.join(str(lookup_es_key(match, k)) for k in self.query_keys())
def related_bucket(self, key):
if key == 'ip' or key.endswith('.ip'):
return 'ip'
if key in self.RELATED_USERS or key.endswith('.user.name'):
return 'user'
if key in self.RELATED_HOSTS:
return 'hosts'
return None
@staticmethod
def valid_ip(value):
try:
ipaddress.ip_address(value)
return True
except ValueError:
return False
def related(self, match):
"""ECS related.* from group-by values; skips invalid IPs."""
related = {}
for key in self.query_keys():
bucket = self.related_bucket(key)
if not bucket:
continue
# original spellings of a lowercased group
value = lookup_es_key(match, f"{key}_spellings")
if value is None:
value = lookup_es_key(match, key)
for v in value if isinstance(value, list) else [value]:
if v is None or (bucket == 'ip' and not self.valid_ip(str(v))):
continue
# dict: ordered and deduped
related.setdefault(bucket, {})[str(v)] = None
return {bucket: list(values) for bucket, values in related.items()}
def event_data(self, match):
"""The match minus the compound query_key field, which ES would map by its last part."""
if not self.rule.get('compound_query_key'):
return match
return {k: v for k, v in match.items() if k != self.rule['query_key']}
@staticmethod
def format_value(value):
if isinstance(value, list):
shown = ', '.join(str(v) for v in value[:3])
return shown if len(value) <= 3 else f"{shown} and {len(value) - 3} more"
return str(value)
@staticmethod
def format_count(value):
if isinstance(value, float) and not value.is_integer():
return f"{value:,.2f}"
if isinstance(value, (int, float)):
return f"{int(value):,}"
return str(value)
@staticmethod
def format_duration(seconds):
for unit, size in (('hour', 3600), ('minute', 60)):
if seconds >= 2 * size:
return f"{seconds // size} {unit}s"
return f"{seconds} second{'' if seconds == 1 else 's'}"
def summary(self, match):
"""One-line correlation summary."""
column, label = self.CORRELATION_COUNTS.get(self.rule['sigma_correlation'], (None, '%count%'))
start = ts_to_dt(match['window_start'])
end = ts_to_dt(match['@timestamp'])
values = {
'count': self.format_count(match.get(column)),
'start': start.strftime('%Y-%m-%d %H:%M:%S UTC'),
'end': end.strftime('%Y-%m-%d %H:%M:%S UTC'),
'duration': self.format_duration(int((end - start).total_seconds())),
}
template = self.rule.get('summary_template')
if not template:
groups = ', '.join(f"{k} %{k}%" for k in self.query_keys())
template = f"{label} for {groups} in %duration%" if groups else f"{label} in %duration%"
def fill(m):
if m[1] in values:
return values[m[1]]
value = lookup_es_key(match, m[1])
# unknown placeholders stay visible so typos show
return m[0] if value is None else self.format_value(value)
return self.PLACEHOLDER.sub(fill, template)
required_options = set(['detection_title', 'sigma_level'])
optional_fields = ['sigma_category', 'sigma_product', 'sigma_service']
def alert(self, matches):
for match in matches:
try:
alert_id = self.alert_id(match)
except MATCH_ERRORS as e:
elastalert_logger.warning("Writing alert for rule %s without a stable id, so a retry may duplicate it: %r",
self.rule['detection_public_id'], e)
alert_id = uuid.uuid4().hex
try:
self.write(alert_id, self.payload(match))
except ElasticsearchException as e:
# EAException makes ElastAlert retry
raise EAException(f"Unable to write the alert to Elasticsearch: {str(e)[:ERROR_TEXT_LIMIT]}") from e
timestamp = strftime("%Y-%m-%d"'T'"%H:%M:%S"'.000Z', gmtime())
headers = {"Content-Type": "application/json"}
def payload(self, match):
rule_info = {
"name": self.rule['detection_title'],
"uuid": self.rule['detection_public_id']
}
creds = None
if 'es_username' in self.rule and 'es_password' in self.rule:
creds = (self.rule['es_username'], self.rule['es_password'])
# Add optional fields if they are present in the rule
for field in self.optional_fields:
rule_key = field.split('_')[-1] # Assumes field format "sigma_<key>"
if field in self.rule:
rule_info[rule_key] = self.rule[field]
# Start building the rule dict
rule_info = {
"name": self.rule['detection_title'],
"uuid": self.rule['detection_public_id']
}
event_info = {
"kind": "alert",
"severity": self.rule['event.severity'],
"module": self.rule['event.module'],
"dataset": self.rule['event.dataset'],
"severity_label": self.rule['sigma_level']
}
# Add optional fields if they are present in the rule
for field in self.optional_fields:
rule_key = field.split('_')[-1] # Assumes field format "sigma_<key>"
if field in self.rule:
rule_info[rule_key] = self.rule[field]
payload = {
"tags": ["alert"],
"rule": rule_info,
"event": event_info,
"sigma_level": self.rule['sigma_level'],
"event_data": self.event_data(match),
"@timestamp": datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%S.000Z')
}
if self.is_correlation:
keys = self.query_keys()
try:
# built before any is added, so a failure adds none
reason = self.summary(match)
labels = {"correlation_group_by": ', '.join(keys), "correlation_group": self.group(match)} if keys else None
related = self.related(match)
except MATCH_ERRORS as e:
elastalert_logger.warning("Writing alert for rule %s without its correlation summary: %r",
self.rule['detection_public_id'], e)
else:
payload["event"]["reason"] = reason
if labels:
payload["labels"] = labels
if related:
payload["related"] = related
return payload
def write(self, alert_id, payload):
try:
self.create(alert_id, payload)
except RequestError as e:
# mapping rejections come from event_data; retry it as text
rejection = str(e)[:ERROR_TEXT_LIMIT]
try:
self.create(alert_id, self.without_event_data(payload, rejection))
except RequestError as again:
elastalert_logger.error("Dropping alert %s for rule %s, rejected by Elasticsearch even without its event data: %s; first rejection: %s",
alert_id, self.rule['detection_public_id'], str(again)[:ERROR_TEXT_LIMIT], rejection)
return
elastalert_logger.warning("Stored alert %s for rule %s with its event data as text, rejected by Elasticsearch: %s",
alert_id, self.rule['detection_public_id'], rejection)
def create(self, alert_id, payload):
try:
self.es.create(index=ALERT_INDEX, id=alert_id, body=payload)
except ConflictError:
pass # a repeat id is already stored
@staticmethod
def without_event_data(payload, rejection):
"""Moves event_data to event.original; the tag keeps Fleet's final pipeline from dropping it."""
fallback = {k: v for k, v in payload.items() if k != 'event_data'}
fallback['event'] = dict(payload['event'], original=json.dumps(payload['event_data'], cls=DateTimeEncoder))
fallback['error'] = {'message': f"event_data rejected by Elasticsearch: {rejection}"}
fallback['tags'] = payload['tags'] + ['preserve_original_event']
return fallback
# Construct the payload with the conditional rule_info
payload = {
"tags": "alert",
"rule": rule_info,
"event": {
"severity": self.rule['event.severity'],
"module": self.rule['event.module'],
"dataset": self.rule['event.dataset'],
"severity_label": self.rule['sigma_level']
},
"sigma_level": self.rule['sigma_level'],
"event_data": match,
"@timestamp": timestamp
}
url = f"https://{self.rule['es_host']}:{self.rule['es_port']}/logs-detections.alerts-so/_doc/"
requests.post(url, data=json.dumps(payload), headers=headers, verify=False, auth=creds)
def get_info(self):
return {'type': 'SecurityOnionESAlerter'}
@@ -1,260 +0,0 @@
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
import copy
from datetime import datetime, timezone
import importlib.util
import json
import os
import unittest
from unittest.mock import MagicMock
from elasticsearch.exceptions import ConflictError, ConnectionError, RequestError
spec = importlib.util.spec_from_file_location('securityonion_es', os.path.join(os.path.dirname(__file__), 'securityonion-es.py'))
es = importlib.util.module_from_spec(spec)
spec.loader.exec_module(es)
BASE_RULE = {
'detection_title': 'Many Failed Network Logons To One Host From One Source',
'detection_public_id': '35a42db6-6629-45af-b8aa-e1fa33c28ef5',
'sigma_level': 'medium',
'sigma_correlation': 'event_count',
'event.severity': 3,
'event.module': 'sigma',
'event.dataset': 'sigma.alert',
'summary_template': '%count% failed network logons to %host.name% from %source.ip% in %duration%',
}
PLAIN_RULE = {k: v for k, v in BASE_RULE.items() if k not in ('sigma_correlation', 'summary_template')}
def correlation_match():
return {
'event_count': 3561,
'window_start': '2026-09-30T18:05:10+00:00',
'@timestamp': '2026-09-30T18:07:53+00:00',
'host': {'name': 'host-01'},
'source': {'ip': '192.0.2.10'},
'_id': '6d1c',
'num_hits': 1,
'num_matches': 1,
}
class TestSecurityOnionESAlerter(unittest.TestCase):
def creates(self, rule, match, effects=None):
"""Run alert(); return (body, id) of each create."""
alerter = es.SecurityOnionESAlerter(rule)
alerter.es = MagicMock()
alerter.es.create.side_effect = effects
alerter.alert([match])
calls = alerter.es.create.call_args_list
self.assertTrue(all(c.kwargs['index'] == 'logs-detections.alerts-so' for c in calls))
return [(json.loads(json.dumps(c.kwargs['body'], cls=es.DateTimeEncoder)), c.kwargs['id']) for c in calls]
def send(self, rule, match):
"""Run alert(); return the payload it wrote and its id."""
(payload, alert_id), = self.creates(rule, match)
return payload, alert_id
def test_compound_query_key_left_out_of_event_data(self):
rule = dict(BASE_RULE, compound_query_key=['host.name', 'source.ip'], query_key='host.name,source.ip')
match = correlation_match()
match['host.name,source.ip'] = 'host-01, 192.0.2.10'
original = copy.deepcopy(match)
payload, _ = self.send(rule, match)
self.assertNotIn('host.name,source.ip', payload['event_data'])
self.assertEqual(payload['event_data']['host'], {'name': 'host-01'})
self.assertEqual(payload['event_data']['source'], {'ip': '192.0.2.10'})
self.assertEqual(payload['rule'], {'name': BASE_RULE['detection_title'], 'uuid': BASE_RULE['detection_public_id'], 'correlation': 'event_count'})
self.assertEqual(payload['labels'], {'correlation_group_by': 'host.name, source.ip', 'correlation_group': 'host-01, 192.0.2.10'})
self.assertEqual(payload['related'], {'hosts': ['host-01'], 'ip': ['192.0.2.10']})
self.assertEqual(payload['event']['kind'], 'alert')
self.assertEqual(payload['event']['reason'], '3,561 failed network logons to host-01 from 192.0.2.10 in 2 minutes')
# ElastAlert reuses the match
self.assertEqual(match, original)
def test_single_query_key(self):
rule = dict(BASE_RULE, query_key='user.name', summary_template='%count% failed SOC logins for %user.name%')
match = {'event_count': 3, 'window_start': '2026-09-30T16:49:52+00:00', '@timestamp': '2026-09-30T16:50:00+00:00',
'user': {'name': 'user@example.invalid'}}
payload, _ = self.send(rule, match)
self.assertEqual(payload['labels'], {'correlation_group_by': 'user.name', 'correlation_group': 'user@example.invalid'})
self.assertEqual(payload['related'], {'user': ['user@example.invalid']})
self.assertEqual(payload['event']['reason'], '3 failed SOC logins for user@example.invalid')
self.assertEqual(payload['event_data'], match)
def test_related_buckets(self):
rule = dict(BASE_RULE, compound_query_key=['winlog.event_data.TargetUserName', 'source.ip', 'dns.highest_registered_domain'],
query_key='winlog.event_data.TargetUserName,source.ip,dns.highest_registered_domain')
match = correlation_match()
match.update({'winlog': {'event_data': {'TargetUserName': ['admin1', 'svc', 'admin1']}},
'source': {'ip': 'not-an-ip'}, 'dns': {'highest_registered_domain': 'example.com'}})
payload, _ = self.send(rule, match)
# deduped; invalid IP skipped; domain stays in the group only
self.assertEqual(payload['related'], {'user': ['admin1', 'svc']})
self.assertEqual(payload['labels']['correlation_group'], "['admin1', 'svc', 'admin1'], not-an-ip, example.com")
payload, _ = self.send(dict(BASE_RULE, query_key='dns.highest_registered_domain'), match)
self.assertNotIn('related', payload)
def test_related_uses_original_spellings(self):
rule = dict(BASE_RULE, query_key='user.name', summary_template=None)
match = {'event_count': 3, 'window_start': '2026-09-30T16:49:52+00:00', '@timestamp': '2026-09-30T16:50:00+00:00',
'user': {'name': 'admin', 'name_spellings': ['Admin', 'admin', 'ADMIN']}}
payload, _ = self.send(rule, match)
# the group shows the lowercased value; related.user finds every spelling
self.assertEqual(payload['labels']['correlation_group'], 'admin')
self.assertEqual(payload['related'], {'user': ['Admin', 'admin', 'ADMIN']})
self.assertEqual(payload['event']['reason'], '3 events for user.name admin in 8 seconds')
def test_plain_rule_has_no_correlation_fields(self):
# a query_key alone (e.g. from an override) isn't a correlation
rule = dict(PLAIN_RULE, query_key='user.name')
# ElastAlert parses @timestamp for EQL hits
match = {'@timestamp': datetime(2026, 9, 30, 16, 50, tzinfo=timezone.utc), '_id': 'abc',
'process': {'name': 'whoami.exe'}, 'user': {'name': 'user'}}
payload, alert_id = self.send(rule, match)
alerter = es.SecurityOnionESAlerter(rule)
self.assertEqual(payload['rule'], {'name': PLAIN_RULE['detection_title'], 'uuid': PLAIN_RULE['detection_public_id']})
self.assertNotIn('labels', payload)
self.assertNotIn('related', payload)
self.assertNotIn('reason', payload['event'])
self.assertEqual(payload['event']['kind'], 'alert')
self.assertEqual(payload['event_data'], dict(match, **{'@timestamp': '2026-09-30T16:50:00+00:00'}))
self.assertEqual(alert_id, alerter.alert_id(match))
self.assertNotEqual(alerter.alert_id(match), alerter.alert_id(dict(match, _id='abd')))
self.assertNotEqual(alerter.alert_id(match), es.SecurityOnionESAlerter(dict(rule, detection_public_id='other')).alert_id(match))
# without an _id, never deduplicated
self.assertNotEqual(alerter.alert_id({'_id': None}), alerter.alert_id({'_id': None}))
def test_ungrouped_correlation_id_ignores_row_hash(self):
rule = dict(BASE_RULE, summary_template=None)
first = {k: v for k, v in correlation_match().items() if k not in ('host', 'source')}
# ES|QL hashes the row into _id, so a later count changes it
later = dict(first, event_count=3600, _id='9f2a')
payload, alert_id = self.send(rule, first)
alerter = es.SecurityOnionESAlerter(rule)
self.assertEqual(alerter.alert_id(first), alerter.alert_id(later))
self.assertNotEqual(alerter.alert_id(first), alerter.alert_id(dict(first, **{'@timestamp': '2026-09-30T18:09:00+00:00'})))
self.assertEqual(alert_id, alerter.alert_id(first))
self.assertEqual(payload['event']['reason'], '3,561 events in 2 minutes')
self.assertNotIn('labels', payload)
def test_temporal_count_column(self):
rule = dict(BASE_RULE, sigma_correlation='temporal', summary_template=None)
match = {'event_type_count': 2, 'window_start': '2026-09-30T16:49:52+00:00', '@timestamp': '2026-09-30T16:50:00+00:00'}
payload, _ = self.send(rule, match)
self.assertEqual(payload['event']['reason'], '2 correlated rules matched in 8 seconds')
def test_grouped_correlation_id_unchanged(self):
"""Ids of alerts already written must not change."""
rule = dict(BASE_RULE, compound_query_key=['host.name', 'source.ip'], query_key='host.name,source.ip')
key = f"{BASE_RULE['detection_public_id']}|2026-09-30T18:07:53+00:00|host-01|192.0.2.10"
self.assertEqual(es.SecurityOnionESAlerter(rule).alert_id(correlation_match()), es.hashlib.sha256(key.encode()).hexdigest())
def test_rejected_event_data_is_kept_as_text(self):
rule = dict(BASE_RULE, query_key='user.name', summary_template='%count% failed SOC logins for %user.name%')
match = {'event_count': 3, 'window_start': '2026-09-30T16:49:52+00:00', '@timestamp': '2026-09-30T16:50:00+00:00',
'user': {'name': 'user@example.invalid'}}
rejected = RequestError(400, 'document_parsing_exception', {})
(first, _), (second, _) = self.creates(rule, match, [rejected, None])
self.assertIn('event_data', first)
self.assertNotIn('event_data', second)
self.assertEqual(json.loads(second['event']['original']), match)
self.assertEqual(second['tags'], ['alert', 'preserve_original_event'])
self.assertEqual(first['tags'], ['alert'])
self.assertTrue(second['error']['message'].startswith('event_data rejected by Elasticsearch: '))
self.assertIn('document_parsing_exception', second['error']['message'])
# everything else carries over
self.assertEqual({k: v for k, v in second['event'].items() if k != 'original'}, first['event'])
changed = ('event_data', 'event', 'error', 'tags')
self.assertEqual({k: v for k, v in second.items() if k not in changed}, {k: v for k, v in first.items() if k not in changed})
def test_rejected_dates_are_kept_as_text(self):
match = {'@timestamp': datetime(2026, 9, 30, 16, 50, tzinfo=timezone.utc), '_id': 'abc'}
_, (second, _) = self.creates(PLAIN_RULE, match, [RequestError(400, 'document_parsing_exception', {}), None])
self.assertEqual(json.loads(second['event']['original']), {'@timestamp': '2026-09-30T16:50:00+00:00', '_id': 'abc'})
def test_rejected_twice_is_dropped_without_retry(self):
rejected = RequestError(400, 'document_parsing_exception', {})
match = {'@timestamp': '2026-09-30T16:50:00+00:00', '_id': 'abc'}
with self.assertLogs('elastalert', 'ERROR') as logs:
self.assertEqual(len(self.creates(PLAIN_RULE, match, [rejected, rejected])), 2)
self.assertIn('Dropping alert', logs.output[0])
def test_write_failure_is_retried(self):
match = {'@timestamp': '2026-09-30T16:50:00+00:00', '_id': 'abc'}
with self.assertRaisesRegex(es.EAException, 'Unable to write the alert to Elasticsearch'):
self.creates(PLAIN_RULE, match, [ConnectionError('N/A', 'refused', None)])
def test_repeat_id_is_ignored(self):
match = {'@timestamp': '2026-09-30T16:50:00+00:00', '_id': 'abc'}
self.assertEqual(len(self.creates(PLAIN_RULE, match, [ConflictError(409, 'version_conflict_engine_exception', {})])), 1)
def test_correlation_fields_are_optional(self):
rule = dict(BASE_RULE, query_key='source.ip')
# no window_start: the summary cannot be built
match = {k: v for k, v in correlation_match().items() if k != 'window_start'}
with self.assertLogs('elastalert', 'WARNING') as logs:
payload, _ = self.send(rule, match)
self.assertIn('without its correlation summary', logs.output[0])
self.assertEqual(payload['event']['kind'], 'alert')
self.assertNotIn('reason', payload['event'])
self.assertNotIn('labels', payload)
self.assertNotIn('related', payload)
def test_unstable_id_still_writes(self):
# no @timestamp: the window end is unknown
match = {k: v for k, v in correlation_match().items() if k not in ('@timestamp', 'window_start')}
with self.assertLogs('elastalert', 'WARNING') as logs:
payload, alert_id = self.send(BASE_RULE, match)
self.assertIn('without a stable id', logs.output[0])
self.assertEqual(len(alert_id), 32)
self.assertEqual(payload['event_data'], match)
def test_summary_formats_values(self):
rule = dict(BASE_RULE, sigma_correlation='value_avg', query_key='source.ip',
summary_template='%count% for %source.ip% to %destination.port% %no.such.field%')
match = {'value_avg': 2.5, 'window_start': '2026-09-30T16:49:52+00:00', '@timestamp': '2026-09-30T16:50:00+00:00',
'source': {'ip': '192.0.2.10'}, 'destination': {'port': [22, 80, 443, 8080, 8443]}}
payload, _ = self.send(rule, match)
self.assertEqual(payload['event']['reason'], '2.50 for 192.0.2.10 to 22, 80, 443 and 2 more %no.such.field%')
self.assertEqual(es.SecurityOnionESAlerter.format_count('n/a'), 'n/a')
def test_get_info(self):
self.assertEqual(es.SecurityOnionESAlerter(PLAIN_RULE).get_info(), {'type': 'SecurityOnionESAlerter'})
+1 -1
View File
@@ -120,7 +120,7 @@ elastalert:
helpLink: elastalert
old_query_limit:
minutes:
description: How long ElastAlert can be down, in minutes, and still resume each rule where it stopped. After a longer outage, rules restart from now and skip the gap.
description: Amount of time in minutes between queries to start at the most recently run query.
global: True
helpLink: elastalert
es_conn_timeout:
@@ -19,7 +19,6 @@ elastic-agent-pr:
- gid: 948
- home: /opt/so/conf/elastic-fleet-pr
- createhome: False
- shell: /sbin/nologin
{% else %}
+2 -3
View File
@@ -20,7 +20,6 @@ elastic-agent:
- gid: 949
- home: /opt/so/conf/elastic-agent
- createhome: False
- shell: /sbin/nologin
elasticagentconfdir:
file.directory:
@@ -40,8 +39,8 @@ elasticagent_sbin_jinja:
file.recurse:
- name: /usr/sbin
- source: salt://elasticagent/tools/sbin_jinja
- user: root
- group: root
- user: 949
- group: 939
- file_mode: 755
- template: jinja
+6 -7
View File
@@ -26,14 +26,13 @@ elastic-fleet:
- gid: 947
- home: /opt/so/conf/elastic-fleet
- createhome: False
- shell: /sbin/nologin
elasticfleet_sbin:
file.recurse:
- name: /usr/sbin
- source: salt://elasticfleet/tools/sbin
- user: root
- group: root
- user: 947
- group: 939
- file_mode: 755
- show_changes: False
@@ -41,8 +40,8 @@ elasticfleet_sbin_jinja:
file.recurse:
- name: /usr/sbin
- source: salt://elasticfleet/tools/sbin_jinja
- user: root
- group: root
- user: 947
- group: 939
- file_mode: 755
- template: jinja
- exclude_pat:
@@ -82,8 +81,8 @@ eapackageupgrade:
file.managed:
- name: /usr/sbin/so-elastic-fleet-package-upgrade
- source: salt://elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade
- user: root
- group: root
- user: 947
- group: 939
- mode: 755
- template: jinja
@@ -29,7 +29,7 @@
"\\.gz$"
],
"include_files": [],
"processors": "- dissect:\n tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"\n- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n- add_fields:\n target: event\n fields:\n dataset: windows.forwarded\n module: windows\n imported: true\n- add_fields:\n target: \"@metadata\"\n fields:\n pipeline: import.evtx\n- if:\n equals:\n winlog.channel: 'Security'\n then: \n - add_fields:\n target: event\n fields:\n dataset: system.security\n module: system\n- if:\n equals:\n winlog.channel: 'Windows PowerShell'\n then: \n - add_fields:\n target: event\n fields:\n dataset: windows.powershell\n module: windows\n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n then: \n - add_fields:\n target: event\n fields:\n dataset: windows.sysmon_operational\n module: windows\n imported: true\n- if:\n equals:\n winlog.channel: 'Application'\n then: \n - add_fields:\n target: event\n fields:\n dataset: system.application\n module: system\n- if:\n equals:\n winlog.channel: 'System'\n then: \n - add_fields:\n target: event\n fields:\n dataset: system.system\n module: system\n \n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n then: \n - add_fields:\n target: event\n fields:\n dataset: windows.powershell_operational\n module: windows\n- add_fields:\n target: data_stream\n fields:\n type: logs\n dataset: import",
"processors": "- dissect:\n tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"\n- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n- drop_fields:\n fields: [\"host\"]\n ignore_missing: true\n- add_fields:\n target: data_stream\n fields:\n type: logs\n dataset: system.security\n- add_fields:\n target: event\n fields:\n dataset: system.security\n module: system\n imported: true\n- add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.security-2.22.3\n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.sysmon_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.sysmon_operational\n module: windows\n imported: true\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.sysmon_operational-3.9.0\n- if:\n equals:\n winlog.channel: 'Application'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.application\n - add_fields:\n target: event\n fields:\n dataset: system.application\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.application-2.22.3\n- if:\n equals:\n winlog.channel: 'System'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.system\n - add_fields:\n target: event\n fields:\n dataset: system.system\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.system-2.22.3\n \n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.powershell_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.powershell_operational\n module: windows\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.powershell_operational-3.9.0\n- add_fields:\n target: data_stream\n fields:\n dataset: import",
"tags": [
"import"
],
+2 -2
View File
@@ -14,8 +14,8 @@ so-elastic-agent-install:
file.managed:
- name: /usr/sbin/so-elastic-agent-install
- source: salt://elasticfleet/tools/sbin/so-elastic-agent-install
- user: root
- group: root
- user: 947
- group: 939
- mode: 755
- show_changes: False
@@ -30,6 +30,8 @@
'azure_metrics.monitor': 'azure.monitor',
'azure_metrics.storage_account': 'azure.storage_account',
'azure_openai.metrics': 'azure.open_ai',
'beat.state': 'beats.stack_monitoring.state',
'beat.stats': 'beats.stack_monitoring.stats',
'enterprisesearch.health': 'enterprisesearch.stack_monitoring.health',
'enterprisesearch.stats': 'enterprisesearch.stack_monitoring.stats',
'kibana.cluster_actions': 'kibana.stack_monitoring.cluster_actions',
@@ -38,8 +40,7 @@
'kibana.node_rules': 'kibana.stack_monitoring.node_rules',
'kibana.stats': 'kibana.stack_monitoring.stats',
'kibana.status': 'kibana.stack_monitoring.status',
'logstash.node': 'logstash.stack_monitoring.node',
'logstash.node_cel': 'logstash.node',
'logstash.node_cel': 'logstash.stack_monitoring.node',
'logstash.node_stats': 'logstash.stack_monitoring.node_stats',
'synthetics.browser': 'synthetics-browser',
'synthetics.browser_network': 'synthetics-browser.network',
@@ -30,56 +30,6 @@ fleet_api() {
curl -sK /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/${QUERYPATH}" "$@" --retry 3 --retry-delay 10 --fail 2>/dev/null
}
elastic_fleet_require_agent_policy() {
local AGENT_POLICY=$1
local POLICY_JSON
if ! POLICY_JSON=$(fleet_api "agent_policies/$AGENT_POLICY") || [ -z "$POLICY_JSON" ]; then
echo "Error: Agent policy '$AGENT_POLICY' was not found or is not visible in the current Kibana space." >&2
return 1
fi
if ! jq -e '.item.package_policies | type == "array"' <<<"$POLICY_JSON" >/dev/null 2>&1; then
echo "Error: Agent policy '$AGENT_POLICY' was not found or is not visible in the current Kibana space." >&2
return 1
fi
echo "$POLICY_JSON"
}
# Print the single active enrollment token for POLICY_ID.
# Exit 1: retryable (API failure, invalid response, no active token)
# Exit 2: multiple active tokens - Shouldn't get into this state without manual intervention
elastic_fleet_active_enrollment_token() {
local POLICY_ID=$1
local RESP TOKEN_COUNT API_KEY
if ! RESP=$(fleet_api "enrollment_api_keys?perPage=100" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'); then
echo "Error: Failed to retrieve enrollment tokens for agent policy '$POLICY_ID'." >&2
return 1
fi
if ! jq -e '.list' <<<"$RESP" >/dev/null 2>&1; then
echo "Error: Invalid enrollment token response for agent policy '$POLICY_ID'." >&2
return 1
fi
TOKEN_COUNT=$(jq --arg pid "$POLICY_ID" '[.list[] | select(.policy_id == $pid and .active == true)] | length' <<<"$RESP")
if [ "${TOKEN_COUNT:-0}" -eq 0 ]; then
echo "Error: No active enrollment token found for agent policy '$POLICY_ID'." >&2
return 1
fi
if [ "$TOKEN_COUNT" -gt 1 ]; then
echo "Error: Found $TOKEN_COUNT active enrollment tokens for agent policy '$POLICY_ID'; expected exactly one." >&2
return 2
fi
API_KEY=$(jq -r --arg pid "$POLICY_ID" '.list[] | select(.policy_id == $pid and .active == true) | .api_key' <<<"$RESP")
echo "$API_KEY"
}
# Max number of concurrent Fleet write jobs (create/update). Override via env if needed.
MAX_FLEET_JOBS=${MAX_FLEET_JOBS:-10}
@@ -112,7 +62,15 @@ elastic_fleet_load_integrations_dir() {
i=0
# Fetch the agent policy a single time; we look up integration ids locally below.
if ! POLICY_JSON=$(elastic_fleet_require_agent_policy "$AGENT_POLICY"); then
if ! POLICY_JSON=$(fleet_api "agent_policies/$AGENT_POLICY"); then
echo "Error: Failed to retrieve agent policy '$AGENT_POLICY'."
rm -f "$FAIL_FILE"
rm -rf "$OUT_DIR"
return 1
fi
if ! jq -e '.item.package_policies' <<<"$POLICY_JSON" >/dev/null 2>&1; then
echo "Error: Invalid agent policy response for '$AGENT_POLICY'."
rm -f "$FAIL_FILE"
rm -rf "$OUT_DIR"
return 1
@@ -166,15 +124,9 @@ elastic_fleet_integration_check() {
JSON_STRING=$2
NAME=$(jq -r .name "$JSON_STRING")
INTEGRATION_ID=""
NAME=$(jq -r .name $JSON_STRING)
local POLICY_JSON
if ! POLICY_JSON=$(elastic_fleet_require_agent_policy "$AGENT_POLICY"); then
return 1
fi
INTEGRATION_ID=$(jq -r --arg n "$NAME" '.item.package_policies[]? | select(.name==$n) | .id' <<<"$POLICY_JSON")
INTEGRATION_ID=$(/usr/sbin/so-elastic-fleet-agent-policy-view "$AGENT_POLICY" | jq -r '.item.package_policies[] | select(.name=="'"$NAME"'") | .id')
}
@@ -196,16 +148,7 @@ elastic_fleet_integration_remove() {
NAME=$2
local POLICY_JSON
if ! POLICY_JSON=$(elastic_fleet_require_agent_policy "$AGENT_POLICY"); then
return 1
fi
INTEGRATION_ID=$(jq -r --arg n "$NAME" '.item.package_policies[]? | select(.name==$n) | .id' <<<"$POLICY_JSON")
if [ -z "$INTEGRATION_ID" ]; then
echo "Error: Integration '$NAME' was not found in agent policy '$AGENT_POLICY'." >&2
return 1
fi
INTEGRATION_ID=$(/usr/sbin/so-elastic-fleet-agent-policy-view "$AGENT_POLICY" | jq -r '.item.package_policies[] | select(.name=="'"$NAME"'") | .id')
JSON_STRING=$( jq -n \
--arg INTEGRATIONID "$INTEGRATION_ID" \
@@ -13,10 +13,7 @@ ERROR=false
for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/elastic-defend/*.json
do
printf "\n\nInitial Endpoints Policy - Loading $INTEGRATION\n"
if ! elastic_fleet_integration_check "endpoints-initial" "$INTEGRATION"; then
ERROR=true
continue
fi
elastic_fleet_integration_check "endpoints-initial" "$INTEGRATION"
if [ -n "$INTEGRATION_ID" ]; then
printf "\n\nIntegration $NAME exists - Upgrading integration policy\n"
if ! elastic_fleet_integration_policy_upgrade "$INTEGRATION_ID"; then
@@ -7,35 +7,20 @@
. /usr/sbin/so-elastic-fleet-common
# Get all the fleet policies
if ! json_output=$(fleet_api "agent_policies" -H 'kbn-xsrf: true'); then
echo "Error: Failed to retrieve Fleet agent policies." >&2
exit 1
fi
if ! jq -e '.items' <<<"$json_output" >/dev/null 2>&1; then
echo "Error: Invalid Fleet agent policies response." >&2
exit 1
fi
json_output=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -L -X GET "localhost:5601/api/fleet/agent_policies" -H 'kbn-xsrf: true')
# Extract the IDs that start with "FleetServer_"
POLICY=$(jq -r '.items[] | select(.id | startswith("FleetServer_")) | .id' <<<"$json_output")
POLICY=$(echo "$json_output" | jq -r '.items[] | select(.id | startswith("FleetServer_")) | .id')
# Iterate over each ID in the POLICY variable
for POLICYNAME in $POLICY; do
printf "\nUpdating Policy: $POLICYNAME\n"
if ! POLICY_JSON=$(elastic_fleet_require_agent_policy "$POLICYNAME"); then
exit 1
fi
INTEGRATION_ID=$(jq -r '.item.package_policies[]? | select(.package.name == "fleet_server") | .id' <<<"$POLICY_JSON")
if [ -z "$INTEGRATION_ID" ]; then
echo "Error: fleet_server integration was not found in agent policy '$POLICYNAME'." >&2
exit 1
fi
# First get the Integration ID
INTEGRATION_ID=$(/usr/sbin/so-elastic-fleet-agent-policy-view "$POLICYNAME" | jq -r '.item.package_policies[] | select(.package.name == "fleet_server") | .id')
# Modify the default integration policy to update the policy_id and an with the correct naming
UPDATED_INTEGRATION_POLICY=$(jq --arg policy_id "$POLICYNAME" --arg name "fleet_server-$POLICYNAME" '
UPDATED_INTEGRATION_POLICY=$(jq --arg policy_id "$POLICYNAME" --arg name "fleet_server-$POLICYNAME" '
.policy_id = $policy_id |
.name = $name' /opt/so/conf/elastic-fleet/integrations/fleet-server/fleet-server.json)
@@ -22,19 +22,12 @@ NUM_RUNNING=$(pgrep -cf "/bin/bash /sbin/so-elastic-agent-gen-installers")
for i in {1..30}
do
ENROLLMENTOKEN=$(elastic_fleet_active_enrollment_token "endpoints-initial")
TOKEN_RC=$?
if [ "$TOKEN_RC" -eq 2 ]; then
exit 1
fi
ENROLLMENTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys?perPage=100" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' | jq .list | jq -r -c '.[] | select(.policy_id | contains("endpoints-initial")) | .api_key')
FLEETHOST=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/fleet_server_hosts/grid-default' | jq -r '.item.host_urls[]' | paste -sd ',')
if [[ -n "$FLEETHOST" ]] && [[ -n "$ENROLLMENTOKEN" ]]; then
break
fi
sleep 10
if [[ $FLEETHOST ]] && [[ $ENROLLMENTOKEN ]]; then break; else sleep 10; fi
done
if [[ -z "$FLEETHOST" ]] || [[ -z "$ENROLLMENTOKEN" ]]; then
if [[ -z $FLEETHOST ]] || [[ -z $ENROLLMENTOKEN ]]; then
printf "\nFleet Host URL, Enrollment Token or Elastic Version empty - exiting..."
printf "\nFleet Host: $FLEETHOST, Enrollment Token: $ENROLLMENTOKEN\n"
exit 1
@@ -74,25 +67,19 @@ for GOOS in "${GOTARGETOS[@]}"; do
GOARCH="amd64"
if [[ $GOOS == 'darwin/arm64' ]]; then GOOS="darwin" && GOARCH="arm64"; fi
printf "\n\n### Generating $GOOS/$GOARCH Installer...\n"
if ! docker run -e CGO_ENABLED=0 -e GOOS=$GOOS -e GOARCH=$GOARCH \
docker run -e CGO_ENABLED=0 -e GOOS=$GOOS -e GOARCH=$GOARCH \
--mount type=bind,source=/etc/pki/tls/certs/,target=/workspace/files/cert/ \
--mount type=bind,source=/nsm/elastic-agent-workspace/,target=/workspace/files/elastic-agent/ \
--mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/,target=/output/ \
{{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} go build -ldflags "-X main.fleetHostURLsList=$FLEETHOST -X main.enrollmentToken=$ENROLLMENTOKEN" -o /output/so-elastic-agent_${GOOS}_${GOARCH}; then
printf "\n### ERROR: Failed to generate $GOOS/$GOARCH installer. Exiting...\n"
exit 1
fi
{{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} go build -ldflags "-X main.fleetHostURLsList=$FLEETHOST -X main.enrollmentToken=$ENROLLMENTOKEN" -o /output/so-elastic-agent_${GOOS}_${GOARCH}
printf "\n### $GOOS/$GOARCH Installer Generated...\n"
done
printf "\n\n### Generating MSI...\n"
cp /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_windows_amd64 /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_windows_amd64.exe
if ! docker run \
docker run \
--mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/,target=/output/ -w /output \
{{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} wixl -o so-elastic-agent_windows_amd64_msi --arch x64 /workspace/so-elastic-agent.wxs; then
printf "\n### ERROR: Failed to generate MSI. Exiting...\n"
exit 1
fi
{{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} wixl -o so-elastic-agent_windows_amd64_msi --arch x64 /workspace/so-elastic-agent.wxs
printf "\n### MSI Generated...\n"
# Verify installers were created
@@ -202,9 +202,26 @@ fi
### Finalization ###
# Query for Enrollment Tokens for default policies
ENDPOINTSENROLLMENTOKEN=$(elastic_fleet_active_enrollment_token "endpoints-initial") || exit 1
GRIDNODESENROLLMENTOKENGENERAL=$(elastic_fleet_active_enrollment_token "so-grid-nodes_general") || exit 1
GRIDNODESENROLLMENTOKENHEAVY=$(elastic_fleet_active_enrollment_token "so-grid-nodes_heavy") || exit 1
if ENDPOINTSENROLLMENTOKEN_RAW=$(fleet_api "enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'); then
ENDPOINTSENROLLMENTOKEN=$(echo "$ENDPOINTSENROLLMENTOKEN_RAW" | jq .list | jq -r -c '.[] | select(.policy_id | contains("endpoints-initial")) | .api_key')
else
echo -e "\nFailed to query for Endpoints enrollment token"
exit 1
fi
if GRIDNODESENROLLMENTOKENGENERAL_RAW=$(fleet_api "enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'); then
GRIDNODESENROLLMENTOKENGENERAL=$(echo "$GRIDNODESENROLLMENTOKENGENERAL_RAW" | jq .list | jq -r -c '.[] | select(.policy_id | contains("so-grid-nodes_general")) | .api_key')
else
echo -e "\nFailed to query for Grid nodes - General enrollment token"
exit 1
fi
if GRIDNODESENROLLMENTOKENHEAVY_RAW=$(fleet_api "enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'); then
GRIDNODESENROLLMENTOKENHEAVY=$(echo "$GRIDNODESENROLLMENTOKENHEAVY_RAW" | jq .list | jq -r -c '.[] | select(.policy_id | contains("so-grid-nodes_heavy")) | .api_key')
else
echo -e "\nFailed to query for Grid nodes - Heavy enrollment token"
exit 1
fi
# Store needed data in minion pillar
pillar_file=/opt/so/saltstack/local/pillar/minions/{{ GLOBALS.minion_id }}.sls
+10 -11
View File
@@ -32,14 +32,13 @@ elasticsearch:
- gid: 930
- home: /opt/so/conf/elasticsearch
- createhome: False
- shell: /sbin/nologin
elasticsearch_sbin:
file.recurse:
- name: /usr/sbin
- source: salt://elasticsearch/tools/sbin
- user: root
- group: root
- user: 930
- group: 939
- file_mode: 755
- exclude_pat:
- so-elasticsearch-pipelines # exclude this because we need to watch it for changes, we sync it in another state
@@ -50,8 +49,8 @@ so-elasticsearch-system-indices-patch-script:
file.managed:
- name: /usr/sbin/so-elasticsearch-system-indices-patch
- source: salt://elasticsearch/tools/sbin/so-elasticsearch-system-indices-patch
- user: root
- group: root
- user: 930
- group: 939
- mode: 755
- show_changes: False
@@ -59,8 +58,8 @@ elasticsearch_sbin_jinja:
file.recurse:
- name: /usr/sbin
- source: salt://elasticsearch/tools/sbin_jinja
- user: root
- group: root
- user: 939
- group: 939
- file_mode: 755
- template: jinja
- exclude_pat:
@@ -73,8 +72,8 @@ so-elasticsearch-ilm-policy-load-script:
file.managed:
- name: /usr/sbin/so-elasticsearch-ilm-policy-load
- source: salt://elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load
- user: root
- group: root
- user: 930
- group: 939
- mode: 754
- template: jinja
- defaults:
@@ -85,8 +84,8 @@ so-elasticsearch-pipelines-script:
file.managed:
- name: /usr/sbin/so-elasticsearch-pipelines
- source: salt://elasticsearch/tools/sbin/so-elasticsearch-pipelines
- user: root
- group: root
- user: 930
- group: 939
- mode: 754
- show_changes: False
+1 -7
View File
@@ -1,7 +1,7 @@
elasticsearch:
enabled: false
esheap: '600m'
version: 9.4.8
version: 9.4.5
index_clean: true
data_retention_method: DLM
vm:
@@ -1160,7 +1160,6 @@ elasticsearch:
- so-fleet_agent_id_verification-1
- so-logs-mappings
- so-logs-settings
- detections-alerts-mappings
data_stream:
allow_custom_routing: false
hidden: false
@@ -3310,7 +3309,6 @@ elasticsearch:
composed_of:
- event-mappings
- logs-system.security@package
- so-fleet_system.security_caseless-1
- logs-system.security@custom
- so-fleet_integrations.ip_mappings-1
- so-fleet_globals-1
@@ -4177,7 +4175,6 @@ elasticsearch:
index_template:
composed_of:
- logs-windows.forwarded@package
- so-fleet_process_caseless-1
- logs-windows.forwarded@custom
- so-fleet_integrations.ip_mappings-1
- so-fleet_globals-1
@@ -4227,7 +4224,6 @@ elasticsearch:
index_template:
composed_of:
- logs-windows.powershell@package
- so-fleet_process_caseless-1
- logs-windows.powershell@custom
- so-fleet_integrations.ip_mappings-1
- so-fleet_globals-1
@@ -4277,7 +4273,6 @@ elasticsearch:
index_template:
composed_of:
- logs-windows.powershell_operational@package
- so-fleet_process_caseless-1
- logs-windows.powershell_operational@custom
- so-fleet_integrations.ip_mappings-1
- so-fleet_globals-1
@@ -4327,7 +4322,6 @@ elasticsearch:
index_template:
composed_of:
- logs-windows.sysmon_operational@package
- so-fleet_process_caseless-1
- logs-windows.sysmon_operational@custom
- so-fleet_integrations.ip_mappings-1
- so-fleet_globals-1
+3 -29
View File
@@ -1,5 +1,5 @@
{
"version": 4,
"version": 3,
"_meta": {
"managed_by": "securityonion",
"managed": true
@@ -99,7 +99,7 @@
},
{
"set": {
"if": "ctx.tags != null && ctx.tags.contains('import') && ctx._index != null && ctx._index.startsWith('logs-import-')",
"if": "ctx.tags != null && ctx.tags.contains('import')",
"override": true,
"field": "data_stream.dataset",
"value": "import"
@@ -107,7 +107,7 @@
},
{
"set": {
"if": "ctx.tags != null && ctx.tags.contains('import') && ctx._index != null && ctx._index.startsWith('logs-import-')",
"if": "ctx.tags != null && ctx.tags.contains('import')",
"override": true,
"field": "data_stream.namespace",
"value": "so"
@@ -177,32 +177,6 @@
"description": "Extract IPs from Elastic Agent events (host.ip) and adds them to related.ip"
}
},
{
"trim": {
"if": "ctx.event?.dataset == 'system.auth'",
"description": "Grok leaves a leading space on 'invalid user' names (elastic/integrations#12174) and, before 2.23.2, sudo padding",
"field": "user.name",
"ignore_missing": true,
"ignore_failure": true
}
},
{
"trim": {
"if": "ctx.event?.dataset == 'system.auth'",
"description": "Appended from the untrimmed user.name",
"field": "related.user",
"ignore_missing": true,
"ignore_failure": true
}
},
{
"script": {
"if": "ctx.event?.dataset == 'system.auth' && ctx.related?.user instanceof List",
"description": "Dedupe after trimming",
"source": "ctx.related.user = new ArrayList(new LinkedHashSet(ctx.related.user));",
"ignore_failure": true
}
},
{
"script": {
"description": "Snapshot event.ingested into _tmp.event_ingested_pre_fleet before .fleet_final_pipeline-1 overwrites it with ES ingest time",
@@ -1,31 +0,0 @@
{
"description" : "import.evtx: normalize imported EVTX and reroute to logs-<dataset>-import",
"processors" : [
{ "script": {
"description": "Host from the event, not the importing node",
"lang": "painless",
"source": "Map host = ['os': ['type': 'windows', 'family': 'windows', 'platform': 'windows']]; def cn = ctx.winlog?.computer_name; if (cn != null && cn.toString().length() > 0) { String name = cn.toString(); int dot = name.indexOf('.'); if (dot > 0) { name = name.substring(0, dot); } host.put('hostname', name); host.put('name', name.toLowerCase()); } ctx.host = host;"
} },
{ "script": {
"description": "String event IDs, as Winlogbeat sends",
"lang": "painless",
"source": "if (ctx.winlog?.event_id != null) { ctx.winlog.event_id = ctx.winlog.event_id.toString(); } if (ctx.event?.code != null) { ctx.event.code = ctx.event.code.toString(); }"
} },
{ "script": {
"description": "Unnamed <Data> to param1..N, as Winlogbeat",
"lang": "painless",
"if": "ctx.winlog?.event_data?.Data instanceof Map && ctx.winlog.event_data.Data['#text'] != null",
"source": "def t = ctx.winlog.event_data.Data['#text']; List vals = t instanceof List ? t : [t]; for (int i = 0; i < vals.size(); i++) { ctx.winlog.event_data['param' + (i + 1)] = vals.get(i); } ctx.winlog.event_data.remove('Data');"
} },
{ "script": {
"description": "String values and LF line endings, as Winlogbeat",
"lang": "painless",
"if": "ctx.winlog?.event_data instanceof Map || ctx.winlog?.user_data instanceof Map",
"source": "String lf = String.valueOf((char) 10); String crlf = String.valueOf((char) 13) + lf; for (def key : ['event_data', 'user_data']) { def m = ctx.winlog[key]; if (!(m instanceof Map)) { continue; } for (def e : m.entrySet()) { def v = e.getValue(); if (v instanceof String) { e.setValue(v.replace(crlf, lf)); } else if (v instanceof Number || v instanceof Boolean) { e.setValue(v.toString()); } } }"
} },
{ "set": { "description": "event.kind, as Winlogbeat", "field": "event.kind", "value": "event", "override": false } },
{ "set": { "field": "data_stream.dataset", "copy_from": "event.dataset", "override": true, "ignore_empty_value": true } },
{ "set": { "field": "data_stream.namespace", "value": "import", "override": true } },
{ "reroute": { "dataset": "{{data_stream.dataset}}", "namespace": "{{data_stream.namespace}}" } }
]
}
@@ -5,8 +5,7 @@
{ "rename": { "field": "message2.proto", "target_field": "network.transport", "ignore_missing": true } },
{ "rename": { "field": "message2.app_proto", "target_field": "network.protocol", "ignore_missing": true } },
{ "rename": { "field": "message2.fileinfo.filename", "target_field": "file.name", "ignore_missing": true } },
{ "rename": { "field": "message2.fileinfo.gaps", "target_field": "suricata.fileinfo.gaps", "ignore_missing": true } },
{ "set": { "if": "ctx.suricata?.fileinfo?.gaps == false", "field": "file.bytes.missing", "value": 0 } },
{ "rename": { "field": "message2.fileinfo.gaps", "target_field": "file.bytes.missing", "ignore_missing": true } },
{ "rename": { "field": "message2.fileinfo.magic", "target_field": "file.mime_type", "ignore_missing": true } },
{ "rename": { "field": "message2.fileinfo.md5", "target_field": "hash.md5", "ignore_missing": true } },
{ "rename": { "field": "message2.fileinfo.sha1", "target_field": "hash.sha1", "ignore_missing": true } },
@@ -1,123 +0,0 @@
{
"_meta": {
"managed_by": "security_onion",
"managed": true,
"description": "Adds .caseless for Lucene queries. Restates each field's package type and .text."
},
"template": {
"mappings": {
"properties": {
"process": {
"properties": {
"executable": {
"type": "keyword",
"ignore_above": 1024,
"fields": {
"caseless": {
"type": "keyword",
"ignore_above": 1024,
"normalizer": "lowercase"
},
"text": {
"type": "match_only_text"
}
}
},
"name": {
"type": "keyword",
"ignore_above": 1024,
"fields": {
"caseless": {
"type": "keyword",
"ignore_above": 1024,
"normalizer": "lowercase"
},
"text": {
"type": "match_only_text"
}
}
},
"command_line": {
"type": "wildcard",
"ignore_above": 1024,
"fields": {
"caseless": {
"type": "keyword",
"ignore_above": 1024,
"normalizer": "lowercase"
},
"text": {
"type": "match_only_text"
}
}
},
"parent": {
"properties": {
"executable": {
"type": "keyword",
"ignore_above": 1024,
"fields": {
"caseless": {
"type": "keyword",
"ignore_above": 1024,
"normalizer": "lowercase"
},
"text": {
"type": "match_only_text"
}
}
},
"name": {
"type": "keyword",
"ignore_above": 1024,
"fields": {
"caseless": {
"type": "keyword",
"ignore_above": 1024,
"normalizer": "lowercase"
},
"text": {
"type": "match_only_text"
}
}
},
"command_line": {
"type": "wildcard",
"ignore_above": 1024,
"fields": {
"caseless": {
"type": "keyword",
"ignore_above": 1024,
"normalizer": "lowercase"
},
"text": {
"type": "match_only_text"
}
}
}
}
}
}
},
"file": {
"properties": {
"path": {
"type": "keyword",
"ignore_above": 1024,
"fields": {
"caseless": {
"type": "keyword",
"ignore_above": 1024,
"normalizer": "lowercase"
},
"text": {
"type": "match_only_text"
}
}
}
}
}
}
}
}
}
@@ -1,80 +0,0 @@
{
"_meta": {
"managed_by": "security_onion",
"managed": true,
"description": "Adds .caseless for Lucene queries. Keeps each field's existing keyword type."
},
"template": {
"mappings": {
"properties": {
"process": {
"properties": {
"command_line": {
"type": "keyword",
"ignore_above": 1024,
"fields": {
"caseless": {
"type": "keyword",
"ignore_above": 1024,
"normalizer": "lowercase"
}
}
},
"parent": {
"properties": {
"executable": {
"type": "keyword",
"ignore_above": 1024,
"fields": {
"caseless": {
"type": "keyword",
"ignore_above": 1024,
"normalizer": "lowercase"
}
}
},
"name": {
"type": "keyword",
"ignore_above": 1024,
"fields": {
"caseless": {
"type": "keyword",
"ignore_above": 1024,
"normalizer": "lowercase"
}
}
},
"command_line": {
"type": "keyword",
"ignore_above": 1024,
"fields": {
"caseless": {
"type": "keyword",
"ignore_above": 1024,
"normalizer": "lowercase"
}
}
}
}
}
}
},
"file": {
"properties": {
"path": {
"type": "keyword",
"ignore_above": 1024,
"fields": {
"caseless": {
"type": "keyword",
"ignore_above": 1024,
"normalizer": "lowercase"
}
}
}
}
}
}
}
}
}
@@ -154,20 +154,6 @@
}
}
}
},
"runtime": {
"so_detection.sigmaKind": {
"type": "keyword",
"script": {
"source": "if (doc['so_detection.language'].size() == 0 || doc['so_detection.language'].value != 'sigma') { return; } def d = params._source.so_detection; if (d == null || d.content == null) { return; } String c = d.content; String NL = String.valueOf((char) 10); emit(c.startsWith('correlation:') || c.contains(NL + 'correlation:') ? 'correlation' : 'single');"
}
},
"so_detection.sigmaCorrelationType": {
"type": "keyword",
"script": {
"source": "if (doc['so_detection.language'].size() == 0 || doc['so_detection.language'].value != 'sigma') { return; } def d = params._source.so_detection; if (d == null || d.content == null) { return; } String c = d.content; String NL = String.valueOf((char) 10); int i = c.startsWith('correlation:') ? 0 : c.indexOf(NL + 'correlation:'); if (i < 0) { return; } i = c.indexOf(NL, i + 1); while (i >= 0 && i + 1 < c.length()) { int e = c.indexOf(NL, i + 1); String line = e < 0 ? c.substring(i + 1) : c.substring(i + 1, e); if (line.length() > 0 && !Character.isWhitespace(line.charAt(0)) && line.charAt(0) != (char) 35) { return; } String t = line.trim(); if (t.startsWith('type:')) { String v = t.substring(5).trim(); int h = v.indexOf(' ' + String.valueOf((char) 35)); if (h >= 0) { v = v.substring(0, h).trim(); } if (v.length() > 1 && (v.charAt(0) == (char) 39 || v.charAt(0) == (char) 34)) { v = v.substring(1, v.length() - 1); } emit(v); return; } i = e; }"
}
}
}
}
},
@@ -1,149 +0,0 @@
{
"template": {
"mappings": {
"properties": {
"tags": {
"ignore_above": 1024,
"type": "keyword"
},
"sigma_level": {
"ignore_above": 1024,
"type": "keyword"
},
"rule": {
"properties": {
"name": {
"ignore_above": 1024,
"type": "keyword"
},
"uuid": {
"ignore_above": 1024,
"type": "keyword"
},
"category": {
"ignore_above": 1024,
"type": "keyword"
},
"product": {
"ignore_above": 1024,
"type": "keyword"
},
"service": {
"ignore_above": 1024,
"type": "keyword"
},
"correlation": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"event": {
"properties": {
"severity": {
"type": "long"
},
"severity_label": {
"ignore_above": 1024,
"type": "keyword"
},
"module": {
"ignore_above": 1024,
"type": "keyword"
},
"dataset": {
"ignore_above": 1024,
"type": "keyword"
},
"kind": {
"ignore_above": 1024,
"type": "keyword"
},
"reason": {
"type": "match_only_text",
"fields": {
"keyword": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"original": {
"type": "keyword",
"index": false,
"doc_values": false
}
}
},
"event_data": {
"properties": {
"@timestamp": {
"type": "date"
},
"window_start": {
"type": "date"
},
"event_count": {
"type": "long"
},
"value_count": {
"type": "long"
},
"event_type_count": {
"type": "long"
},
"value_sum": {
"type": "double"
},
"value_avg": {
"type": "double"
},
"value_percentile": {
"type": "double"
},
"value_median": {
"type": "double"
}
}
},
"labels": {
"properties": {
"correlation_group_by": {
"ignore_above": 1024,
"type": "keyword"
},
"correlation_group": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"related": {
"properties": {
"ip": {
"type": "ip"
},
"user": {
"ignore_above": 1024,
"type": "keyword"
},
"hosts": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"error": {
"properties": {
"message": {
"type": "match_only_text"
}
}
}
}
}
},
"_meta": {
"description": "Fields written by the ElastAlert SecurityOnionESAlerter to logs-detections.alerts-so"
}
}
+14 -33
View File
@@ -4,19 +4,11 @@
{%- set role = GLOBALS.role.split('-')[1] %}
{%- from 'firewall/containers.map.jinja' import NODE_CONTAINERS %}
{%- set NODE_NETWORKS = [] %}
{%- for NETNAME, NETWORK in DOCKERMERGED.networks.items() %}
{%- if not NETWORK.get('manager_only') or GLOBALS.get('is_manager', False) %}
{%- do NODE_NETWORKS.append(NETNAME) %}
{%- endif %}
{%- endfor %}
{%- set PR = [] %}
{%- set D1 = [] %}
{%- set D2 = [] %}
{%- for container in NODE_CONTAINERS %}
{%- set IP = DOCKERMERGED.containers[container].ip %}
{%- set BRIDGE = DOCKERMERGED.containers[container].network %}
{%- if DOCKERMERGED.containers[container].port_bindings is defined %}
{%- for binding in DOCKERMERGED.containers[container].port_bindings %}
{#- cant split int so we convert to string #}
@@ -43,11 +35,11 @@
{%- endif %}
{%- do PR.append("-A POSTROUTING -s " ~ DOCKERMERGED.containers[container].ip ~ "/32 -d " ~ DOCKERMERGED.containers[container].ip ~ "/32 -p " ~ proto ~ " -m " ~ proto ~ " --dport " ~ containerPort ~ " -j MASQUERADE") %}
{%- if bindip | length and bindip != '0.0.0.0' %}
{%- do D1.append("-A DOCKER -d " ~ bindip ~ "/32 ! -i " ~ BRIDGE ~ " -p " ~ proto ~ " -m " ~ proto ~ " --dport " ~ hostPort ~ " -j DNAT --to-destination " ~ DOCKERMERGED.containers[container].ip ~ ":" ~ containerPort) %}
{%- do D1.append("-A DOCKER -d " ~ bindip ~ "/32 ! -i sobridge -p " ~ proto ~ " -m " ~ proto ~ " --dport " ~ hostPort ~ " -j DNAT --to-destination " ~ DOCKERMERGED.containers[container].ip ~ ":" ~ containerPort) %}
{%- else %}
{%- do D1.append("-A DOCKER ! -i " ~ BRIDGE ~ " -p " ~ proto ~ " -m " ~ proto ~ " --dport " ~ hostPort ~ " -j DNAT --to-destination " ~ DOCKERMERGED.containers[container].ip ~ ":" ~ containerPort) %}
{%- do D1.append("-A DOCKER ! -i sobridge -p " ~ proto ~ " -m " ~ proto ~ " --dport " ~ hostPort ~ " -j DNAT --to-destination " ~ DOCKERMERGED.containers[container].ip ~ ":" ~ containerPort) %}
{%- endif %}
{%- do D2.append("-A DOCKER -d " ~ DOCKERMERGED.containers[container].ip ~ "/32 ! -i " ~ BRIDGE ~ " -o " ~ BRIDGE ~ " -p " ~ proto ~ " -m " ~ proto ~ " --dport " ~ containerPort ~ " -j ACCEPT") %}
{%- do D2.append("-A DOCKER -d " ~ DOCKERMERGED.containers[container].ip ~ "/32 ! -i sobridge -o sobridge -p " ~ proto ~ " -m " ~ proto ~ " --dport " ~ containerPort ~ " -j ACCEPT") %}
{%- endfor %}
{%- endif %}
{%- endfor %}
@@ -60,15 +52,11 @@
:DOCKER - [0:0]
-A PREROUTING -m addrtype --dst-type LOCAL -j DOCKER
-A OUTPUT ! -d 127.0.0.0/8 -m addrtype --dst-type LOCAL -j DOCKER
{%- for NETNAME in NODE_NETWORKS %}
-A POSTROUTING -s {{ DOCKERMERGED.networks[NETNAME].range }} ! -o {{ NETNAME }} -j MASQUERADE
{%- endfor %}
-A POSTROUTING -s {{DOCKERMERGED.range}} ! -o sobridge -j MASQUERADE
{%- for rule in PR %}
{{ rule }}
{%- endfor %}
{%- for NETNAME in NODE_NETWORKS %}
-A DOCKER -i {{ NETNAME }} -j RETURN
{%- endfor %}
-A DOCKER -i sobridge -j RETURN
{%- for rule in D1 %}
{{ rule }}
{%- endfor %}
@@ -109,12 +97,10 @@ COMMIT
{%- endif %}
-A FORWARD -j DOCKER-USER
-A FORWARD -j DOCKER-ISOLATION-STAGE-1
{%- for NETNAME in NODE_NETWORKS %}
-A FORWARD -o {{ NETNAME }} -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
-A FORWARD -o {{ NETNAME }} -j DOCKER
-A FORWARD -i {{ NETNAME }} ! -o {{ NETNAME }} -j ACCEPT
-A FORWARD -i {{ NETNAME }} -o {{ NETNAME }} -j ACCEPT
{%- endfor %}
-A FORWARD -o sobridge -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
-A FORWARD -o sobridge -j DOCKER
-A FORWARD -i sobridge ! -o sobridge -j ACCEPT
-A FORWARD -i sobridge -o sobridge -j ACCEPT
-A FORWARD -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
-A FORWARD -i lo -j ACCEPT
-A FORWARD -m conntrack --ctstate INVALID -j DROP
@@ -126,18 +112,13 @@ COMMIT
{%- for rule in D2 %}
{{ rule }}
{%- endfor %}
{% for NETNAME in NODE_NETWORKS %}
-A DOCKER-ISOLATION-STAGE-1 -i {{ NETNAME }} ! -o {{ NETNAME }} -j DOCKER-ISOLATION-STAGE-2
{%- endfor %}
-A DOCKER-ISOLATION-STAGE-1 -i sobridge ! -o sobridge -j DOCKER-ISOLATION-STAGE-2
-A DOCKER-ISOLATION-STAGE-1 -j RETURN
{%- for NETNAME in NODE_NETWORKS %}
-A DOCKER-ISOLATION-STAGE-2 -o {{ NETNAME }} -j DROP
{%- endfor %}
-A DOCKER-ISOLATION-STAGE-2 -o sobridge -j DROP
-A DOCKER-ISOLATION-STAGE-2 -j RETURN
{%- for NETNAME in NODE_NETWORKS %}
-A DOCKER-USER ! -i {{ NETNAME }} -o {{ NETNAME }} -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
-A DOCKER-USER ! -i {{ NETNAME }} -o {{ NETNAME }} -j LOGGING
{%- endfor %}
-A DOCKER-USER ! -i sobridge -o sobridge -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
-A DOCKER-USER ! -i sobridge -o sobridge -j LOGGING
-A DOCKER-USER -j RETURN
-A LOGGING -m limit --limit 2/min -j LOG --log-prefix "IPTables-dropped: "
-A LOGGING -j DROP
+2 -6
View File
@@ -4,12 +4,8 @@
{# add our ip to self #}
{% do FIREWALL_DEFAULT.firewall.hostgroups.self.append(GLOBALS.node_ip) %}
{# add dockernet ranges #}
{% for NETNAME, NETWORK in DOCKERMERGED.networks.items() %}
{% if not NETWORK.get('manager_only') or GLOBALS.get('is_manager', False) %}
{% do FIREWALL_DEFAULT.firewall.hostgroups.dockernet.append(NETWORK.range) %}
{% endif %}
{% endfor %}
{# add dockernet range #}
{% do FIREWALL_DEFAULT.firewall.hostgroups.dockernet.append(DOCKERMERGED.range) %}
{% if GLOBALS.role == 'so-idh' %}
{% from 'idh/opencanary_config.map.jinja' import IDH_PORTGROUPS %}
+3 -3
View File
@@ -26,8 +26,8 @@ so-hydra:
- hostname: hydra
- name: so-hydra
- networks:
- soauth:
- ipv4_address: {{ DOCKERMERGED.containers['so-hydra'].ips['soauth'] }}
- sobridge:
- ipv4_address: {{ DOCKERMERGED.containers['so-hydra'].ip }}
- binds:
- /opt/so/conf/hydra/:/hydra-conf:ro
- /opt/so/log/hydra/:/hydra-log:rw
@@ -73,7 +73,7 @@ delete_so-hydra_so-status.disabled:
wait_for_hydra:
http.wait_for_successful_query:
- name: 'http://{{ DOCKERMERGED.containers['so-hydra'].ips['soauth'] }}:4444/health/alive'
- name: 'http://{{ GLOBALS.manager }}:4444/health/alive'
- ssl: True
- verify_ssl: False
- status:
-4
View File
@@ -21,16 +21,12 @@ hypervisor_sbin:
file.recurse:
- name: /usr/sbin
- source: salt://hypervisor/tools/sbin
- user: root
- group: root
- file_mode: 744
hypervisor_sbin_jinja:
file.recurse:
- name: /usr/sbin
- source: salt://hypervisor/tools/sbin_jinja
- user: root
- group: root
- template: jinja
- file_mode: 744
+2 -2
View File
@@ -86,8 +86,8 @@ idh_sbin:
file.recurse:
- name: /usr/sbin
- source: salt://idh/tools/sbin
- user: root
- group: root
- user: 939
- group: 939
- file_mode: 755
#idh_sbin_jinja:
+2 -2
View File
@@ -41,8 +41,8 @@ influxdb_sbin:
file.recurse:
- name: /usr/sbin
- source: salt://influxdb/tools/sbin
- user: root
- group: root
- user: 939
- group: 939
- file_mode: 755
#influxdb_sbin_jinja:
+1 -3
View File
@@ -94,11 +94,9 @@ metrics_link_file:
- docker_container: so-influxdb
# Install cron job to determine size of influxdb for telegraf
# telegraf reads this while the cron rewrites it, so write aside and rename rather than
# truncating in place. tgraflogdir recurses ownership, so the temp file is chowned to match
get_influxdb_size:
cron.present:
- name: 'du -s -k /nsm/influxdb | cut -f1 > /opt/so/log/telegraf/influxdb_size.log.tmp 2>&1; chown 939:939 /opt/so/log/telegraf/influxdb_size.log.tmp; mv -f /opt/so/log/telegraf/influxdb_size.log.tmp /opt/so/log/telegraf/influxdb_size.log'
- name: 'du -s -k /nsm/influxdb | cut -f1 > /opt/so/log/telegraf/influxdb_size.log 2>&1'
- identifier: get_influxdb_size
- user: root
- minute: '*/1'
+4 -5
View File
@@ -21,7 +21,6 @@ kafka_user:
- gid: 960
- home: /opt/so/conf/kafka
- createhome: False
- shell: /sbin/nologin
kafka_home_dir:
file.absent:
@@ -31,16 +30,16 @@ kafka_sbin_tools:
file.recurse:
- name: /usr/sbin
- source: salt://kafka/tools/sbin
- user: root
- group: root
- user: 960
- group: 960
- file_mode: 755
kafka_sbin_jinja_tools:
file.recurse:
- name: /usr/sbin
- source: salt://kafka/tools/sbin_jinja
- user: root
- group: root
- user: 960
- group: 960
- file_mode: 755
- template: jinja
- defaults:
+4 -5
View File
@@ -22,7 +22,6 @@ kibana:
- gid: 932
- home: /opt/so/conf/kibana
- createhome: False
- shell: /sbin/nologin
# Drop the correct nginx config based on role
@@ -37,16 +36,16 @@ kibana_sbin:
file.recurse:
- name: /usr/sbin
- source: salt://kibana/tools/sbin
- user: root
- group: root
- user: 932
- group: 939
- file_mode: 755
kibana_sbin_jinja:
file.recurse:
- name: /usr/sbin
- source: salt://kibana/tools/sbin_jinja
- user: root
- group: root
- user: 932
- group: 939
- file_mode: 755
- template: jinja
- defaults:
+1 -1
View File
@@ -22,7 +22,7 @@ kibana:
- default
- file
migrations:
discardCorruptObjects: "9.4.8"
discardCorruptObjects: "9.4.5"
telemetry:
enabled: False
xpack:
-1
View File
@@ -27,7 +27,6 @@ kratos:
- uid: 928
- gid: 928
- home: /opt/so/conf/kratos
- shell: /sbin/nologin
kratosdir:
file.directory:
+3 -3
View File
@@ -19,8 +19,8 @@ so-kratos:
- hostname: kratos
- name: so-kratos
- networks:
- soauth:
- ipv4_address: {{ DOCKERMERGED.containers['so-kratos'].ips['soauth'] }}
- sobridge:
- ipv4_address: {{ DOCKERMERGED.containers['so-kratos'].ip }}
- binds:
- /opt/so/conf/kratos/:/kratos-conf:ro
- /opt/so/log/kratos/:/kratos-log:rw
@@ -71,7 +71,7 @@ delete_so-kratos_so-status.disabled:
wait_for_kratos:
http.wait_for_successful_query:
- name: 'http://{{ DOCKERMERGED.containers['so-kratos'].ips['soauth'] }}:4434/'
- name: 'http://{{ GLOBALS.manager }}:4434/'
- ssl: True
- verify_ssl: False
- status:
-2
View File
@@ -6,8 +6,6 @@ so-fix-salt-ldap_script:
file.managed:
- name: /usr/sbin/so-fix-salt-ldap.py
- source: salt://libvirt/64962/scripts/so-fix-salt-ldap.py
- user: root
- group: root
- mode: 744
fix-salt-ldap:
+2 -3
View File
@@ -35,14 +35,13 @@ logstash:
- uid: 931
- gid: 931
- home: /opt/so/conf/logstash
- shell: /sbin/nologin
logstash_sbin:
file.recurse:
- name: /usr/sbin
- source: salt://logstash/tools/sbin
- user: root
- group: root
- user: 931
- group: 939
- file_mode: 755
#logstash_sbin_jinja:
+8 -12
View File
@@ -113,8 +113,8 @@ manager_sbin:
file.recurse:
- name: /usr/sbin
- source: salt://manager/tools/sbin
- user: root
- group: root
- user: 939
- group: 939
- file_mode: 755
- exclude_pat:
- "*_test.py"
@@ -124,8 +124,8 @@ manager_sbin_jinja:
file.recurse:
- name: /usr/sbin/
- source: salt://manager/tools/sbin_jinja/
- user: root
- group: root
- user: socore
- group: socore
- file_mode: 755
- template: jinja
- show_changes: False
@@ -166,7 +166,7 @@ so-repo-sync:
so_fleetagent_status:
cron.present:
- name: '/usr/sbin/so-elasticagent-status > /opt/so/log/agents/agentstatus.log.tmp 2>&1; mv -f /opt/so/log/agents/agentstatus.log.tmp /opt/so/log/agents/agentstatus.log'
- name: /usr/sbin/so-elasticagent-status > /opt/so/log/agents/agentstatus.log 2>&1
- identifier: so_fleetagent_status
- user: root
- minute: '*/5'
@@ -190,15 +190,11 @@ so_fleetagent_monitor:
- month: '*'
- dayweek: '*'
# This tree is the source of every root-executed script (/usr/sbin, reactors, _runners,
# engines, salt-relay.sh). SOC mounts /opt/so/saltstack rw as uid 939 but only writes
# under local/. Do not add dir_mode/file_mode here -- SOC reads default/ and 750/640
# would break its config load.
root_own_saltstack_default:
socore_own_saltstack_default:
file.directory:
- name: /opt/so/saltstack/default
- user: root
- group: root
- user: socore
- group: socore
- recurse:
- user
- group
+8 -13
View File
@@ -106,8 +106,7 @@ while [[ $# -gt 0 ]]; do
esac
done
hydraContainer=${HYDRA_CONTAINER:-so-hydra}
hydraUrl=${HYDRA_URL:-http://localhost:4445}
hydraUrl=${HYDRA_URL:-http://127.0.0.1:4445}
socRolesFile=${SOC_ROLES_FILE:-/opt/so/conf/soc/soc_clients_roles}
soUID=${SOCORE_UID:-939}
soGID=${SOCORE_GID:-939}
@@ -125,10 +124,6 @@ function fail() {
exit 1
}
function hydraCurl() {
docker exec "$hydraContainer" curl "$@"
}
function require() {
cmd=$1
which "$1" 2>&1 > /dev/null
@@ -138,8 +133,8 @@ function require() {
# Verify this environment is capable of running this script
function verifyEnvironment() {
require "jq"
require "docker"
response=$(hydraCurl -Ss -L ${hydraUrl}/health/alive)
require "curl"
response=$(curl -Ss -L ${hydraUrl}/health/alive)
[[ "$response" != '{"status":"ok"}' ]] && fail "Unable to communicate with Hydra; specify URL via HYDRA_URL environment variable"
}
@@ -169,7 +164,7 @@ function ensureRoleFileExists() {
}
function listClients() {
response=$(hydraCurl -Ss -L -f ${hydraUrl}/admin/clients)
response=$(curl -Ss -L -f ${hydraUrl}/admin/clients)
[[ $? != 0 ]] && fail "Unable to communicate with Hydra"
clientIds=$(echo "${response}" | jq -r ".[] | .client_id" | sort)
@@ -256,7 +251,7 @@ function createClient() {
EOF
)
response=$(hydraCurl -Ss -L --fail-with-body -X POST ${hydraUrl}/admin/clients -d "$body")
response=$(curl -Ss -L --fail-with-body -X POST ${hydraUrl}/admin/clients -d "$body")
if [[ $? != 0 ]]; then
error=$(echo $response | jq .error)
fail "Failed to submit request to Hydra: $error"
@@ -288,7 +283,7 @@ function update() {
EOF
)
response=$(hydraCurl -Ss -L --fail-with-body -X PATCH ${hydraUrl}/admin/clients/$id -d "$body")
response=$(curl -Ss -L --fail-with-body -X PATCH ${hydraUrl}/admin/clients/$id -d "$body")
if [[ $? != 0 ]]; then
error=$(echo $response | jq .error)
fail "Failed to submit request to Hydra: $error"
@@ -310,7 +305,7 @@ function generateSecret() {
EOF
)
response=$(hydraCurl -Ss -L --fail-with-body -X PATCH ${hydraUrl}/admin/clients/$id -d "$body")
response=$(curl -Ss -L --fail-with-body -X PATCH ${hydraUrl}/admin/clients/$id -d "$body")
if [[ $? != 0 ]]; then
error=$(echo $response | jq .error)
fail "Failed to submit request to Hydra: $error"
@@ -322,7 +317,7 @@ function deleteClient() {
[[ ${identityId} == "" ]] && fail "Client not found"
response=$(hydraCurl -Ss -XDELETE -L --fail-with-body "${hydraUrl}/admin/clients/$identityId")
response=$(curl -Ss -XDELETE -L --fail-with-body "${hydraUrl}/admin/clients/$identityId")
if [[ $? != 0 ]]; then
error=$(echo $response | jq .error)
fail "Failed to submit request to Hydra: $error"
+6 -84
View File
@@ -121,14 +121,8 @@ for i in "$@"; do
esac
done
if [[ -n "$MINION_ID" && ! "$MINION_ID" =~ ^[A-Za-z0-9._-]{1,253}$ ]]; then
echo "Invalid minion id: $MINION_ID"
log "ERROR" "Invalid minion id: $MINION_ID"
exit 1
fi
readonly PILLARFILE=/opt/so/saltstack/local/pillar/minions/$MINION_ID.sls
readonly ADVPILLARFILE=/opt/so/saltstack/local/pillar/minions/adv_$MINION_ID.sls
PILLARFILE=/opt/so/saltstack/local/pillar/minions/$MINION_ID.sls
ADVPILLARFILE=/opt/so/saltstack/local/pillar/minions/adv_$MINION_ID.sls
function getinstallinfo() {
log "INFO" "Getting install info for minion $MINION_ID"
@@ -139,23 +133,10 @@ function getinstallinfo() {
return 1
fi
# install.txt is controlled by the minion; only accept known keys and never eval or export them
local line key
while IFS= read -r line; do
[[ "$line" == *=* ]] || continue
key=${line%%=*}
case "$key" in
MAINIP|MNIC|NODE_DESCRIPTION|ES_HEAP_SIZE|PATCHSCHEDULENAME|INTERFACE|NODETYPE|CORECOUNT|LSHOSTNAME|LSHEAP|CPUCORES|IDH_MGTRESTRICT|IDH_SERVICES)
printf -v "$key" '%s' "${line#*=}"
;;
*)
log "WARN" "Ignoring unexpected install var from $MINION_ID: ${key:0:64}"
;;
esac
done <<< "$INSTALLVARS"
if [[ "$NODE_DESCRIPTION" == \'*\' ]]; then
NODE_DESCRIPTION=${NODE_DESCRIPTION:1:-1}
while read -r var; do export "$var"; done <<< "$INSTALLVARS"
if [ $? -ne 0 ]; then
log "ERROR" "Failed to source install variables"
return 1
fi
log "INFO" "Fetched install info for $MINION_ID (node type: ${NODETYPE:-unset})"
@@ -195,12 +176,6 @@ function pcapspace() {
fi
fi
# Must be checked before arithmetic expansion, which evaluates array subscripts
if [[ ! "$SPACESIZE" =~ ^[0-9]+$ ]]; then
log "ERROR" "Invalid disk size for $MINION_ID: ${SPACESIZE:0:64}"
return 1
fi
local s=$(( $SPACESIZE / 1000000 ))
local s1=$(( $s / 4 * $PCAP_PERCENTAGE ))
@@ -1075,57 +1050,6 @@ function updateMineAndApplyStates() {
fi
}
# Values end up in a Jinja-rendered pillar and in bash, and may come from the minion
function validate_minion_vars() {
local error_msg=""
# Inline rather than valid_ip4: so-common is not installed yet when setup runs -o=setup
local octet='(25[0-5]|2[0-4][0-9]|1?[0-9]?[0-9])'
local ip4_re="^($octet\.){3}$octet$"
case "$NODETYPE" in
EVAL|STANDALONE|MANAGER|MANAGERSEARCH|MANAGERHYPE|IMPORT)
# Manager pillars also rewrite the CA pillar, so never accept them from a remote node
[[ "$OPERATION" == "setup" ]] || error_msg="Node type $NODETYPE can only be configured during setup"
;;
FLEET|IDH|HEAVYNODE|SENSOR|SEARCHNODE|RECEIVER|HYPERVISOR|DESKTOP)
;;
*)
error_msg="Invalid node type: ${NODETYPE:0:64}"
;;
esac
if [[ -z "$error_msg" ]]; then
if [[ ! "$MAINIP" =~ $ip4_re ]]; then
error_msg="Invalid MAINIP: ${MAINIP:0:64}"
elif [[ ! "$MNIC" =~ ^[A-Za-z0-9._-]*$ ]]; then
error_msg="Invalid MNIC: ${MNIC:0:64}"
elif [[ ! "$INTERFACE" =~ ^[A-Za-z0-9._-]*$ ]]; then
error_msg="Invalid INTERFACE: ${INTERFACE:0:64}"
elif [[ ! "$LSHOSTNAME" =~ ^[A-Za-z0-9._-]*$ ]]; then
error_msg="Invalid LSHOSTNAME: ${LSHOSTNAME:0:64}"
elif [[ ! "$ES_HEAP_SIZE" =~ ^([0-9]+[kKmMgG]?)?$ ]]; then
error_msg="Invalid ES_HEAP_SIZE: ${ES_HEAP_SIZE:0:64}"
elif [[ ! "$LSHEAP" =~ ^([0-9]+[kKmMgG]?)?$ ]]; then
error_msg="Invalid LSHEAP: ${LSHEAP:0:64}"
elif [[ ! "$CORECOUNT" =~ ^[0-9]*$ ]]; then
error_msg="Invalid CORECOUNT: ${CORECOUNT:0:64}"
elif [[ ! "$CPUCORES" =~ ^[0-9]*$ ]]; then
error_msg="Invalid CPUCORES: ${CPUCORES:0:64}"
elif [[ ! "$IDH_MGTRESTRICT" =~ ^(True|False)?$ ]]; then
error_msg="Invalid IDH_MGTRESTRICT: ${IDH_MGTRESTRICT:0:64}"
fi
fi
if [[ -n "$error_msg" ]]; then
log "ERROR" "$error_msg"
echo "$error_msg"
return 1
fi
# Free text; removing braces is enough to prevent any Jinja delimiter
NODE_DESCRIPTION=${NODE_DESCRIPTION//[\{\}[:cntrl:]]/}
}
function setupMinionFiles() {
log "INFO" "Setting up minion files for $MINION_ID (pillar: $PILLARFILE)"
@@ -1137,8 +1061,6 @@ function setupMinionFiles() {
return 1
fi
validate_minion_vars || return 1
# Create the base minion files
create_minion_files || return 1
+2 -2
View File
@@ -124,8 +124,8 @@ copy_new_files() {
rsync -a salt $default_salt_dir/
rsync -a pillar $default_salt_dir/
chown -R root:root $default_salt_dir/salt
chown -R root:root $default_salt_dir/pillar
chown -R socore:socore $default_salt_dir/salt
chown -R socore:socore $default_salt_dir/pillar
chmod 755 $default_salt_dir/pillar/firewall/addfirewall.sh
rm -rf /tmp/sogh
+11 -16
View File
@@ -129,8 +129,7 @@ while [[ $# -gt 0 ]]; do
esac
done
kratosContainer=${KRATOS_CONTAINER:-so-kratos}
kratosUrl=${KRATOS_URL:-http://localhost:4434/admin}
kratosUrl=${KRATOS_URL:-http://127.0.0.1:4434/admin}
databasePath=${KRATOS_DB_PATH:-/nsm/kratos/db/db.sqlite}
databaseTimeout=${KRATOS_DB_TIMEOUT:-5000}
bcryptRounds=${BCRYPT_ROUNDS:-12}
@@ -155,10 +154,6 @@ function fail() {
exit 1
}
function kratosCurl() {
docker exec "$kratosContainer" curl "$@"
}
function require() {
cmd=$1
which "$1" 2>&1 > /dev/null
@@ -169,18 +164,18 @@ function require() {
function verifyEnvironment() {
require "htpasswd"
require "jq"
require "docker"
require "curl"
require "openssl"
require "sqlite3"
[[ ! -f $databasePath ]] && fail "Unable to find database file; specify path via KRATOS_DB_PATH environment variable"
response=$(kratosCurl -Ss -L ${kratosUrl}/)
response=$(curl -Ss -L ${kratosUrl}/)
[[ "$response" != "404 page not found" ]] && fail "Unable to communicate with Kratos; specify URL via KRATOS_URL environment variable"
}
function findIdByEmail() {
email=${1,,}
response=$(kratosCurl -Ss -L ${kratosUrl}/identities)
response=$(curl -Ss -L ${kratosUrl}/identities)
identityId=$(echo "${response}" | jq -r ".[] | select(.verifiable_addresses[0].value == \"$email\") | .id")
echo $identityId
}
@@ -421,7 +416,7 @@ function syncAll() {
}
function listUsers() {
response=$(kratosCurl -Ss -L ${kratosUrl}/identities)
response=$(curl -Ss -L ${kratosUrl}/identities)
[[ $? != 0 ]] && fail "Unable to communicate with Kratos"
users=$(echo "${response}" | jq -r ".[] | .verifiable_addresses[0].value" | sort)
@@ -500,7 +495,7 @@ function createUser() {
EOF
)
response=$(kratosCurl -Ss -L ${kratosUrl}/identities -d "$addUserJson")
response=$(curl -Ss -L ${kratosUrl}/identities -d "$addUserJson")
[[ $? != 0 ]] && fail "Unable to communicate with Kratos"
identityId=$(echo "${response}" | jq -r ".id")
@@ -523,7 +518,7 @@ function updateStatus() {
identityId=$(findIdByEmail "$email")
[[ ${identityId} == "" ]] && fail "User not found"
response=$(kratosCurl -Ss -L "${kratosUrl}/identities/$identityId")
response=$(curl -Ss -L "${kratosUrl}/identities/$identityId")
[[ $? != 0 ]] && fail "Unable to communicate with Kratos"
schemaId=$(echo "$response" | jq -r .schema_id)
@@ -536,7 +531,7 @@ function updateStatus() {
state="inactive"
fi
body="{ \"schema_id\": \"$schemaId\", \"state\": \"$state\", \"traits\": $traitBlock }"
response=$(kratosCurl -fSsL -XPUT -H "Content-Type: application/json" "${kratosUrl}/identities/$identityId" -d "$body")
response=$(curl -fSsL -XPUT -H "Content-Type: application/json" "${kratosUrl}/identities/$identityId" -d "$body")
[[ $? != 0 ]] && fail "Unable to update user"
}
@@ -555,7 +550,7 @@ function updateUserProfile() {
identityId=$(findIdByEmail "$email")
[[ ${identityId} == "" ]] && fail "User not found"
response=$(kratosCurl -Ss -L "${kratosUrl}/identities/$identityId")
response=$(curl -Ss -L "${kratosUrl}/identities/$identityId")
[[ $? != 0 ]] && fail "Unable to communicate with Kratos"
schemaId=$(echo "$response" | jq -r .schema_id)
@@ -564,7 +559,7 @@ function updateUserProfile() {
traitBlock="{\"email\":\"$email\",\"firstName\":\"$firstName\",\"lastName\":\"$lastName\",\"note\":\"$note\"}"
body="{ \"schema_id\": \"$schemaId\", \"state\": \"$state\", \"traits\": $traitBlock }"
response=$(kratosCurl -fSsL -XPUT -H "Content-Type: application/json" "${kratosUrl}/identities/$identityId" -d "$body")
response=$(curl -fSsL -XPUT -H "Content-Type: application/json" "${kratosUrl}/identities/$identityId" -d "$body")
[[ $? != 0 ]] && fail "Unable to update user"
}
@@ -574,7 +569,7 @@ function deleteUser() {
identityId=$(findIdByEmail "$email")
[[ ${identityId} == "" ]] && fail "User not found"
response=$(kratosCurl -Ss -XDELETE -L "${kratosUrl}/identities/$identityId")
response=$(curl -Ss -XDELETE -L "${kratosUrl}/identities/$identityId")
[[ $? != 0 ]] && fail "Unable to communicate with Kratos"
rolesTmpFile="${socRolesFile}.tmp"
+1 -2
View File
@@ -42,8 +42,7 @@ def loadYaml(filename):
try:
with open(filename, "r") as file:
content = file.read()
loaded = yaml.safe_load(content)
return loaded if loaded is not None else {}
return yaml.safe_load(content)
except FileNotFoundError:
print(f"File not found: {filename}", file=sys.stderr)
sys.exit(1)
-97
View File
@@ -95,20 +95,6 @@ class TestRemove(unittest.TestCase):
expected = "key1:\n child1: 123\n child2:\n deep2: ab\nkey2: false\n"
self.assertEqual(actual, expected)
def test_remove_empty_file(self):
filename = "/tmp/so-yaml_test-remove-empty.yaml"
file = open(filename, "w")
file.close()
code = soyaml.remove([filename, "key1"])
self.assertEqual(code, 0)
file = open(filename, "r")
actual = file.read()
file.close()
self.assertEqual(actual, "{}\n")
def test_remove_missing_args(self):
with patch('sys.exit', new=MagicMock()) as sysmock:
with patch('sys.stderr', new=StringIO()) as mock_stderr:
@@ -308,36 +294,6 @@ class TestRemove(unittest.TestCase):
expected = "key1:\n child1: 123\n child2:\n deep1: 45\n deep2: d\nkey2: false\nkey3:\n- e\n- f\n- g\n"
self.assertEqual(actual, expected)
def test_add_empty_file(self):
filename = "/tmp/so-yaml_test-add-empty.yaml"
file = open(filename, "w")
file.close()
code = soyaml.add([filename, "telegraf.output", "BOTH"])
self.assertEqual(code, 0)
file = open(filename, "r")
actual = file.read()
file.close()
expected = "telegraf:\n output: BOTH\n"
self.assertEqual(actual, expected)
def test_add_empty_file_simple(self):
filename = "/tmp/so-yaml_test-add-empty-simple.yaml"
file = open(filename, "w")
file.close()
code = soyaml.add([filename, "telegraf", "BOTH"])
self.assertEqual(code, 0)
file = open(filename, "r")
actual = file.read()
file.close()
expected = "telegraf: BOTH\n"
self.assertEqual(actual, expected)
def test_replace_missing_arg(self):
with patch('sys.exit', new=MagicMock()) as sysmock:
with patch('sys.stderr', new=StringIO()) as mock_stderr:
@@ -390,21 +346,6 @@ class TestRemove(unittest.TestCase):
expected = "key1:\n child1: 123\n child2:\n deep1: 46\nkey2: false\nkey3:\n- e\n- f\n- g\n"
self.assertEqual(actual, expected)
def test_replace_empty_file(self):
filename = "/tmp/so-yaml_test-replace-empty.yaml"
file = open(filename, "w")
file.close()
code = soyaml.replace([filename, "telegraf.output", "BOTH"])
self.assertEqual(code, 0)
file = open(filename, "r")
actual = file.read()
file.close()
expected = "telegraf:\n output: BOTH\n"
self.assertEqual(actual, expected)
def test_convert(self):
self.assertEqual(soyaml.convertType("foo"), "foo")
self.assertEqual(soyaml.convertType("foo.bar"), "foo.bar")
@@ -565,18 +506,6 @@ class TestRemove(unittest.TestCase):
self.assertEqual(result, 2)
self.assertEqual("", mock_stdout.getvalue())
def test_get_empty_file(self):
with patch('sys.stdout', new=StringIO()) as mock_stdout:
with patch('sys.stderr', new=StringIO()) as mock_stderr:
filename = "/tmp/so-yaml_test-get-empty.yaml"
file = open(filename, "w")
file.close()
result = soyaml.get([filename, "telegraf.output"])
self.assertEqual(result, 2)
self.assertEqual("", mock_stdout.getvalue())
self.assertIn("Key 'telegraf.output' not found by so-yaml.py", mock_stderr.getvalue())
def test_get_usage(self):
with patch('sys.exit', new=MagicMock()) as sysmock:
with patch('sys.stderr', new=StringIO()) as mock_stderr:
@@ -1062,29 +991,3 @@ class TestLoadYaml(unittest.TestCase):
soyaml.loadYaml("/tmp/so-yaml_test-unreadable.yaml")
sysmock.assert_called_with(1)
self.assertIn("Error reading file", mock_stderr.getvalue())
def test_load_yaml_empty_file(self):
filename = "/tmp/so-yaml_test-load-empty.yaml"
file = open(filename, "w")
file.close()
result = soyaml.loadYaml(filename)
self.assertEqual(result, {})
def test_load_yaml_whitespace_only(self):
filename = "/tmp/so-yaml_test-load-whitespace.yaml"
file = open(filename, "w")
file.write(" \n\n \n")
file.close()
result = soyaml.loadYaml(filename)
self.assertEqual(result, {})
def test_load_yaml_comments_only(self):
filename = "/tmp/so-yaml_test-load-comments.yaml"
file = open(filename, "w")
file.write("# Just a comment\n# Another comment\n")
file.close()
result = soyaml.loadYaml(filename)
self.assertEqual(result, {})
+11 -115
View File
@@ -28,7 +28,6 @@ INSTALLEDSALTVERSION=$(salt --versions-report | grep Salt: | awk '{print $2}')
# percentage like "25%"). Empty means so-soup-grid-highstate uses the salt:auto_apply:batch
# pillar default.
BATCHSIZE=
DEFAULT_DOCKER_RANGE='172.17.1.0/24'
SOUP_LOG=/root/soup.log
SOUP_DEBUG_LOG=/root/soup-debug.log
WHATWOULDYOUSAYYAHDOHERE=soup
@@ -606,7 +605,6 @@ preupgrade_changes() {
[[ "$INSTALLEDVERSION" == "3.0.0" ]] && up_to_3.1.0
[[ "$INSTALLEDVERSION" == "3.1.0" ]] && up_to_3.2.0
[[ "$INSTALLEDVERSION" == "3.2.0" ]] && up_to_3.3.0
[[ "$INSTALLEDVERSION" == "3.3.0" ]] && up_to_3.4.0
true
}
@@ -625,7 +623,6 @@ postupgrade_changes() {
[[ "$POSTVERSION" == "3.0.0" ]] && post_to_3.1.0
[[ "$POSTVERSION" == "3.1.0" ]] && post_to_3.2.0
[[ "$POSTVERSION" == "3.2.0" ]] && post_to_3.3.0
[[ "$POSTVERSION" == "3.3.0" ]] && post_to_3.4.0
# All applicable post-upgrade steps completed; clear the resume marker.
rm -f "$POSTVERSION_FILE"
true
@@ -1131,6 +1128,9 @@ post_to_3.2.0() {
### 3.3.0 Scripts ###
up_to_3.3.0() {
# download 9.4.5 elastic agent packages
determine_elastic_agent_upgrade
# remove existing (patched) elasticsearch index template to match integration naming change
if ! remove_elasticsearch_index_template "so-logs-sentinel_one_cloud_funnel.login" "sentinel_one_cloud_funnel.login changed to sentinel_one_cloud_funnel.logins"; then
FINAL_MESSAGE_QUEUE+=("WARNING: Unable to automatically remove the so-logs-sentinel_one_cloud_funnel.login index template. This step can be performed manually using the following command:")
@@ -1162,120 +1162,17 @@ telegraf_repair() {
post_to_3.3.0() {
# Recollate again since some internal DBs were excluded during 3.2.0 soup
recollate_postgres
# Generate 9.4.5 elastic agent installers
echo "Regenerating Elastic Agent Installers"
/sbin/so-elastic-agent-gen-installers
telegraf_repair
set_postversion 3.3.0
}
### 3.3.0 End ###
### 3.4.0 Scripts ###
up_to_3.4.0() {
# download 9.4.8 elastic agent packages
determine_elastic_agent_upgrade
set_soauth_range
echo "Removing so-kratos, so-hydra and so-soc so they are recreated on the soauth network."
docker rm -f so-kratos so-hydra so-soc >> $SOUP_LOG 2>&1
for template in so-metrics-logstash.node so-metrics-logstash.stack_monitoring.node; do
if ! remove_elasticsearch_index_template "$template" "logstash node and node_cel index patterns reversed"; then
FINAL_MESSAGE_QUEUE+=("WARNING: Unable to automatically remove the $template index template. Addon integration templates may fail to load until it is removed:")
FINAL_MESSAGE_QUEUE+=(" - sudo so-elasticsearch-query _index_template/$template -XDELETE && so-checkin")
fi
done
INSTALLEDVERSION=3.4.0
}
set_soauth_range() {
local pillar_file=/opt/so/saltstack/local/pillar/docker/soc_docker.sls
local current_range suggested authnet authgw input
[[ -f "$pillar_file" ]] || return 0
current_range=$(so-yaml.py get -r "$pillar_file" docker.range 2>/dev/null) || return 0
# A default range gets the 172.17.2.0/24 from docker/defaults.yaml, same as a fresh
# install, so there is nothing to ask about.
[[ -n "$current_range" && "$current_range" != "$DEFAULT_DOCKER_RANGE" ]] || return 0
if so-yaml.py get -r "$pillar_file" docker.networks.soauth.range >/dev/null 2>&1; then
return 0
fi
suggested=$(echo "${current_range%%/*}" | awk -F'.' '{ printf "%s.%s.%s.%s", $1, $2, ($3 + 1) % 256, $4 }')
if [[ -z $UNATTENDED ]]; then
echo ""
echo "This grid uses a custom Docker range ($current_range). The authentication"
echo "services are moving to their own isolated network, which needs a second /24"
echo "that does not overlap it."
echo ""
while :; do
read -rp "Enter the network without the /24 suffix, or press Enter for ${suggested}: " input
[[ -z "$input" ]] && input="$suggested"
if valid_soauth_range "$input" "$current_range"; then
authnet="$input"
break
fi
echo "That range must be a valid IPv4 network, must not be within 172.17.0.0/24, and must not overlap ${current_range}."
done
else
if ! valid_soauth_range "$suggested" "$current_range"; then
FINAL_MESSAGE_QUEUE+=("WARNING: Unable to pick a range for the authentication network alongside $current_range. Set it manually before the next highstate:")
FINAL_MESSAGE_QUEUE+=(" - so-yaml.py add $pillar_file docker.networks.soauth.range <network>/24")
FINAL_MESSAGE_QUEUE+=(" - so-yaml.py add $pillar_file docker.networks.soauth.gateway <gateway>")
return 0
fi
authnet="$suggested"
FINAL_MESSAGE_QUEUE+=("NOTE: The authentication services moved to an isolated Docker network and were assigned ${authnet}/24.")
FINAL_MESSAGE_QUEUE+=(" - If that conflicts with your environment, update docker.networks.soauth in $pillar_file and run so-checkin.")
fi
authgw=$(echo "$authnet" | awk -F'.' '{print $1,$2,$3,1}' OFS='.')
echo "Assigning the authentication network the range ${authnet}/24."
so-yaml.py add "$pillar_file" docker.networks.soauth.range "${authnet}/24" >> $SOUP_LOG 2>&1
so-yaml.py add "$pillar_file" docker.networks.soauth.gateway "$authgw" >> $SOUP_LOG 2>&1
}
valid_soauth_range() {
local candidate=$1 docker_range=$2
valid_ip4 "$candidate" || return 1
[[ $candidate =~ ^172\.17\.0\. ]] && return 1
[[ "${candidate}/24" == "$docker_range" ]] && return 1
return 0
}
post_to_3.4.0() {
# Generate 9.4.8 elastic agent installers
echo "Regenerating Elastic Agent Installers"
/sbin/so-elastic-agent-gen-installers
for idx in "metrics-logstash.node-default" "metrics-logstash.stack_monitoring.node-default"; do
rollover_index "$idx"
done
add_detection_runtime_fields
set_postversion 3.4.0
}
add_detection_runtime_fields() {
local template=$DEFAULT_SALT_DIR/salt/elasticsearch/templates/component/so/detection-mappings.json
local runtime
runtime=$(jq -c '{runtime: .template.mappings.runtime}' "$template")
if ! so-elasticsearch-query so-detection/_mapping -XPUT -d "$runtime" --retry 3 --retry-delay 5 --fail >> $SOUP_LOG 2>&1; then
FINAL_MESSAGE_QUEUE+=("WARNING: Unable to add the Sigma search fields to the so-detection index. Searches on so_detection.sigmaKind will match nothing until they are added:")
FINAL_MESSAGE_QUEUE+=(" - sudo so-elasticsearch-query so-detection/_mapping -XPUT -d \"\$(jq -c '{runtime: .template.mappings.runtime}' $template)\"")
fi
}
### 3.4.0 End ###
repo_sync() {
echo "Sync the local repo."
@@ -1548,10 +1445,9 @@ verify_es_version_compatibility() {
["8.18.4"]="8.18.6 8.18.8 9.0.8"
["8.18.6"]="8.18.8 9.0.8"
["8.18.8"]="9.0.8"
["9.0.8"]="9.3.3 9.3.7 9.4.5 9.4.8"
["9.3.3"]="9.3.7 9.4.5 9.4.8"
["9.3.7"]="9.4.5 9.4.8"
["9.4.5"]="9.4.8"
["9.0.8"]="9.3.3 9.3.7 9.4.5"
["9.3.3"]="9.3.7 9.4.5"
["9.3.7"]="9.4.5"
)
# Elasticsearch MUST upgrade through these versions
+2 -2
View File
@@ -57,8 +57,8 @@ nginx_sbin:
file.recurse:
- name: /usr/sbin
- source: salt://nginx/tools/sbin
- user: root
- group: root
- user: 939
- group: 939
- file_mode: 755
#nginx_sbin_jinja:
+1 -1
View File
@@ -183,7 +183,7 @@ http {
ssl_prefer_server_ciphers on;
ssl_protocols TLSv1.2 TLSv1.3;
location ~* (^/login|^/login/.*|^/js/.*|^/css/.*|^/images/.*|^/pages/.*|^/docs/.*) {
location ~* (^/login/.*|^/js/.*|^/css/.*|^/images/.*|^/pages/.*|^/docs/.*) {
proxy_pass http://{{ GLOBALS.manager }}:9822;
proxy_read_timeout 90;
proxy_connect_timeout 90;
+4 -4
View File
@@ -50,16 +50,16 @@ redis_sbin:
file.recurse:
- name: /usr/sbin
- source: salt://redis/tools/sbin
- user: root
- group: root
- user: 939
- group: 939
- file_mode: 755
redis_sbin_jinja:
file.recurse:
- name: /usr/sbin
- source: salt://redis/tools/sbin_jinja
- user: root
- group: root
- user: 939
- group: 939
- file_mode: 755
- template: jinja
+1 -1
View File
@@ -9,7 +9,7 @@
'epel-testing.repo',
'saltstack.repo',
'salt-latest.repo',
'wazuh.repo',
'wazuh.repo'
'Rocky-Base.repo',
'Rocky-CR.repo',
'Rocky-Debuginfo.repo',
+2 -4
View File
@@ -3,8 +3,6 @@ salt_bootstrap:
file.managed:
- name: /usr/sbin/bootstrap-salt.sh
- source: salt://salt/scripts/bootstrap-salt.sh
- user: root
- group: root
- mode: 755
- show_changes: False
@@ -12,6 +10,6 @@ salt_sbin:
file.recurse:
- name: /usr/sbin
- source: salt://salt/tools/sbin
- user: root
- group: root
- user: 939
- group: 939
- file_mode: 755
-2
View File
@@ -35,8 +35,6 @@ combine_bond_script:
file.managed:
- name: /usr/sbin/so-combine-bond
- source: salt://sensor/tools/sbin_jinja/so-combine-bond
- user: root
- group: root
- mode: 755
- template: jinja
- defaults:
+2 -2
View File
@@ -64,8 +64,8 @@ sensoroni_sbin:
file.recurse:
- name: /usr/sbin
- source: salt://sensoroni/tools/sbin
- user: root
- group: root
- user: 939
- group: 939
- file_mode: 755
#sensoroni_sbin_jinja:
@@ -1,3 +1,2 @@
requests>=2.34.0
whoisit>=4.0.5
anyio>=4.15.1
requests>=2.31.0
whoisit>=2.7.0
+8 -20
View File
@@ -118,33 +118,21 @@ crondetectionsbackup:
- month: '*'
- dayweek: '*'
# sigma-cli only loads *.yml from the pipelines dir
socsigmafinalpipeline:
file.managed:
- name: /opt/so/conf/soc/sigma_pipelines/sigma_final_pipeline.yml
- name: /opt/so/conf/soc/sigma_final_pipeline.yaml
- source: salt://soc/files/soc/sigma_final_pipeline.yaml
- user: 939
- group: 939
- mode: 600
- makedirs: True
# sigma-cli loads every *.yml here; clean removes anything else
socsigmapipelines:
file.recurse:
- name: /opt/so/conf/soc/sigma_pipelines
- source: salt://soc/files/soc/sigma_pipelines
socsigmasopipeline:
file.managed:
- name: /opt/so/conf/soc/sigma_so_pipeline.yaml
- source: salt://soc/files/soc/sigma_so_pipeline.yaml
- user: 939
- group: 939
- file_mode: 600
- clean: True
- require:
- file: socsigmafinalpipeline
socsigmapipelinesold:
file.absent:
- names:
- /opt/so/conf/soc/sigma_final_pipeline.yaml
- /opt/so/conf/soc/sigma_so_pipeline.yaml
- mode: 600
socsigmaplaybookpipeline:
file.managed:
@@ -183,8 +171,8 @@ soc_sbin:
file.recurse:
- name: /usr/sbin
- source: salt://soc/tools/sbin
- user: root
- group: root
- user: 939
- group: 939
- file_mode: 755
#soc_sbin_jinja:
-2
View File
@@ -14,8 +14,6 @@
{% do SOCDEFAULTS.soc.config.server.modules[module].update({'hostUrl': application_url}) %}
{% endfor %}
{% do SOCDEFAULTS.soc.config.server.modules.kratos.update({'publicHostUrl': 'http://' ~ DOCKERMERGED.containers['so-kratos'].ips['soauth'] ~ ':4433/'}) %}
{# add all grid heavy nodes to soc.server.modules.elastic.remoteHostUrls #}
{% for node_type, minions in salt['pillar.get']('elasticsearch:nodes', {}).items() %}
{% if node_type in ['heavynode'] %}
+7 -152
View File
@@ -1380,7 +1380,6 @@ soc:
retryFailureMaxAttempts: 5
kratos:
hostUrl:
publicHostUrl:
hydra:
hostUrl:
elastalertengine:
@@ -1445,7 +1444,7 @@ soc:
default:
- repo: https://github.com/Security-Onion-Solutions/securityonion-resources
license: Elastic-2.0
folder: sigma
folder: sigma/stable
community: true
rulesetName: securityonion-resources
- repo: file:///nsm/rules/custom-local-repos/local-sigma
@@ -1455,7 +1454,7 @@ soc:
airgap:
- repo: file:///nsm/rules/detect-sigma/repos/securityonion-resources
license: Elastic-2.0
folder: sigma
folder: sigma/stable
community: true
rulesetName: securityonion-resources
- repo: file:///nsm/rules/custom-local-repos/local-sigma
@@ -1466,9 +1465,6 @@ soc:
- core
- emerging_threats_addon
useEsql: false
esqlCaseInsensitive: true
esqlQueryDelaySeconds: 30
esqlCorrelationAllowanceSeconds: 600
elastic:
hostUrl:
remoteHostUrls: []
@@ -1496,9 +1492,6 @@ soc:
org: Security Onion
bucket: telegraf/so_short_term
verifyCert: false
notification:
dismissedPruneDays: 30
enabled: true
playbook:
autoUpdateEnabled: true
playbookImportFrequencySeconds: 86400
@@ -1542,10 +1535,8 @@ soc:
agentic: false
agentMapping:
Orchestrator: sonnet@SOAI
Investigator: haiku@SOAI
DetectionEngineer: haiku@SOAI
AlertTriage: haiku@SOAI
Notifier: haiku@SOAI
Investigator: gemma@SOAI
DetectionEngineer: gemma@SOAI
useMemory: false
useMemoryScanner: false
dontScanBefore: ""
@@ -1565,77 +1556,6 @@ soc:
reconcilePersona: ""
toolUseTurnAttempts: 12
toolUseTurnDelayMs: 175
agentSessionMaxTurns: 20
agentStreamFlushIntervalMs: 1000
agentStreamIdleTimeoutSeconds: 300
automationSettings:
tickIntervalSeconds: 60
maxConcurrentItems: 4
maxQueuedItems: 0
alertTriageEpoch: "2026-09-24T00:00:00Z"
tools:
filterEventFields:
- "@timestamp"
- "client.name"
- "destination.ip"
- "destination.port"
- "destination.geo.country_name"
- "dns.query.name"
- "dns.query_name"
- "event.action"
- "event.category"
- "event.module"
- "event.dataset"
- "event.outcome"
- "event.severity"
- "event.severity_label"
- "event.type"
- "event_data.agent.name"
- "event_data.host.os.name"
- "file.mime_type"
- "file.name"
- "hash.md5"
- "hash.sha1"
- "host.mac"
- "host.name"
- "host.os.name"
- "http.method"
- "http.useragent"
- "http.virtual_host"
- "log.id.uid"
- "network.community_id"
- "network.protocol"
- "network.transport"
- "notice.message"
- "observer.name"
- "process.name"
- "process.executable"
- "process.entity_id"
- "process.command_line"
- "process.Ext.ancestry"
- "process.parent.entity_id"
- "process.parent.command_line"
- "rule.category"
- "rule.name"
- "rule.uuid"
- "software.name"
- "software.type"
- "software.version.unparsed"
- "source.ip"
- "source.port"
- "source.geo.country_name"
- "ssh.cypher_algorithm"
- "ssh.client"
- "ssh.server"
- "ssl.cipher"
- "ssl.server_name"
- "ssl.version"
- "system.auth.sudo.command"
- "user.name"
- "user.domain"
- "user.effective.name"
- "weird.name"
- "tags"
onionconfig:
saltstackDir: /opt/so/saltstack
bypassEnabled: false
@@ -1825,7 +1745,6 @@ soc:
cacheExpirationMs: 300000
casesEnabled: true
detectionsEnabled: true
allowExternalMarkdownImages: false
inactiveTools: ['toolUnused']
exportNodeId:
tools:
@@ -2684,11 +2603,8 @@ soc:
query: "so_detection.language:suricata | groupby so_detection.ruleset so_detection.isEnabled | groupby so_detection.category"
description: Show all NIDS Detections, which are run with Suricata
- name: "Detection Type - Sigma (Elastalert) - All"
query: "so_detection.language:sigma | groupby so_detection.sigmaKind | groupby so_detection.ruleset so_detection.isEnabled | groupby so_detection.category | groupby so_detection.product"
query: "so_detection.language:sigma | groupby so_detection.ruleset so_detection.isEnabled | groupby so_detection.category | groupby so_detection.product"
description: Show all Sigma Detections, which are run with Elastalert
- name: "Detection Type - Sigma (Elastalert) - Correlations"
query: "so_detection.sigmaKind:correlation | groupby so_detection.sigmaCorrelationType so_detection.isEnabled | groupby so_detection.ruleset"
description: Show Sigma correlation Detections
- name: "Detection Type - YARA (Strelka)"
query: "so_detection.language:yara | groupby so_detection.ruleset so_detection.isEnabled"
description: Show all YARA detections, which are used by Strelka
@@ -2772,7 +2688,7 @@ soc:
elastalert: |
# This is a Sigma rule template, which uses YAML. Replace all template values with your own values.
# The id (UUIDv4) is pregenerated and can safely be used.
# Click "Convert" to convert the Sigma rule to use Security Onion field mappings within a backend query
# Click "Convert" to convert the Sigma rule to use Security Onion field mappings within an EQL query
#
# Rule Creation Guide: https://github.com/SigmaHQ/sigma/wiki/Rule-Creation-High%E2%80%90Level-Guide
# Logsources: https://sigmahq.io/docs/basics/log-sources.html
@@ -2802,58 +2718,6 @@ soc:
- ' -priv'
condition: all of selection_*
level: 'high' # info | low | medium | high | critical
elastalert_correlation: |
# Sigma correlation rule; requires ES|QL (useEsql).
# First document: the correlation. Following documents: the rules it references.
#
# Types: event_count, value_count, temporal, value_sum, value_avg, value_median, value_percentile.
# Correlation Guide: https://sigmahq.io/docs/meta/correlations.html
# Logsources: https://sigmahq.io/docs/basics/log-sources.html
title: 'A Short Capitalized Title With Less Than 50 Characters'
id: [publicId]
status: 'experimental'
description: |
Describe what the correlation finds and, importantly, why relating these
events is more meaningful than either of them alone.
references:
- 'https://local.invalid'
author: '@SecurityOnion'
date: '[today]'
tags:
- detection.threat_hunting
- attack.technique_id
correlation:
type: value_count
rules:
- example_base_rule # the 'name' of the rule below
group-by:
- source.ip
# Xs, Xm, Xh, Xd or Xw.
timespan: 10m
condition:
field: dns.query.name
gte: 40
falsepositives:
- 'Describe the benign activity that also produces this pattern'
# Placeholders: %count%, %start%, %end%, %duration%, or any field.
summary: '%count% distinct names queried by %source.ip% in %duration%'
level: 'medium' # info | low | medium | high | critical
---
title: 'Base Event'
# referenced by 'name' (or 'id')
name: example_base_rule
description: 'The single event that the correlation aggregates.'
logsource:
category: network
service: dns
detection:
selection:
dns.query.name|exists: true
condition: selection
# Carried into the alert.
fields:
- dns.query.name
assistant:
enabled: false
investigationPrompt: Investigate Alert ID {socId}
@@ -2867,16 +2731,7 @@ soc:
- id: sonnet
displayName: Claude Sonnet
origin: USA
contextLimitSmall: 1000000
contextLimitLarge: 1000000
lowBalanceColorAlert: 500000
enabled: true
adapter: SOAI
charsPerTokenEstimate: 4
- id: haiku
displayName: Claude Haiku
origin: USA
contextLimitSmall: 1000000
contextLimitSmall: 200000
contextLimitLarge: 1000000
lowBalanceColorAlert: 500000
enabled: true
+2 -2
View File
@@ -18,8 +18,8 @@ hypervisor_annotation:
- name: /opt/so/saltstack/default/salt/hypervisor/soc_hypervisor.yaml
- source: salt://soc/dyanno/hypervisor/soc_hypervisor.yaml.jinja
- template: jinja
- user: root
- group: root
- user: socore
- group: socore
- defaults:
HYPERVISORS: {{ HYPERVISORS }}
baseDomainStatus: {{ salt['pillar.get']('baseDomain:status', 'Initialized') }}
+3 -5
View File
@@ -23,9 +23,7 @@ so-soc:
- name: so-soc
- networks:
- sobridge:
- ipv4_address: {{ DOCKERMERGED.containers['so-soc'].ips['sobridge'] }}
- soauth:
- ipv4_address: {{ DOCKERMERGED.containers['so-soc'].ips['soauth'] }}
- ipv4_address: {{ DOCKERMERGED.containers['so-soc'].ip }}
- binds:
- /nsm/rules:/nsm/rules:rw
- /opt/so/conf/strelka:/opt/sensoroni/yara:rw
@@ -47,8 +45,9 @@ so-soc:
{% endif %}
- /opt/so/conf/soc/motd.md:/opt/sensoroni/html/motd.md:ro
- /opt/so/conf/soc/banner.md:/opt/sensoroni/html/login/banner.md:ro
- /opt/so/conf/soc/sigma_pipelines:/opt/sensoroni/sigma_pipelines:ro
- /opt/so/conf/soc/sigma_so_pipeline.yaml:/opt/sensoroni/sigma_so_pipeline.yaml:ro
- /opt/so/conf/soc/sigma_playbook_pipeline.yaml:/opt/sensoroni/sigma_playbook_pipeline.yaml:ro
- /opt/so/conf/soc/sigma_final_pipeline.yaml:/opt/sensoroni/sigma_final_pipeline.yaml:ro
- /opt/so/conf/soc/playbook_placeholder_map.yaml:/opt/sensoroni/playbook_placeholder_map.yaml:ro
- /opt/so/conf/soc/playbook_placeholder_map_custom.yaml:/opt/sensoroni/playbook_placeholder_map_custom.yaml:ro
- /opt/so/conf/soc/custom.js:/opt/sensoroni/html/js/custom.js:ro
@@ -106,7 +105,6 @@ so-soc:
- file: socclientsroles
- file: socplaybookplaceholdermap
- file: socplaybookplaceholdermapcustom
- file: socsigmapipelines
delete_so-soc_so-status.disabled:
file.uncomment:
@@ -1,484 +0,0 @@
name: Security Onion ES|QL Pipeline
# ES|QL query settings
priority: 92
transformations:
- id: esql_default_index
type: set_state
key: index
val: .ds-logs-*
- id: esql_source_metadata
type: set_state
key: metadata
val: "_id, _index, _source"
- id: esql_source_keep
type: set_state
key: keep
val: "_id, _index, _source"
# unmapped fields read as null instead of failing the query
- id: esql_unmapped_fields
type: set_state
key: unmapped_fields
val: nullify
# FROM targets per logsource, any namespace; later entries win, correlations get the union
- id: esql_index_process_creation
type: set_state
key: index
val:
- .ds-logs-endpoint.events.process-*
- .ds-logs-windows.sysmon_operational-*
- .ds-logs-system.security-*
- .ds-logs-windows.powershell-*
- .ds-logs-windows.forwarded-*
- .ds-logs-sysmon_linux.log-*
- .ds-logs-auditd_manager.auditd-*
- .ds-logs-import-so-*
rule_conditions:
- type: logsource
category: process_creation
- id: esql_index_process_creation_windows
type: set_state
key: index
val:
- .ds-logs-endpoint.events.process-*
- .ds-logs-windows.sysmon_operational-*
- .ds-logs-system.security-*
- .ds-logs-windows.powershell-*
- .ds-logs-windows.forwarded-*
- .ds-logs-import-so-*
rule_conditions:
- type: logsource
product: windows
category: process_creation
- id: esql_index_process_creation_linux
type: set_state
key: index
val:
- .ds-logs-endpoint.events.process-*
- .ds-logs-sysmon_linux.log-*
- .ds-logs-auditd_manager.auditd-*
- .ds-logs-import-so-*
rule_conditions:
- type: logsource
product: linux
category: process_creation
- id: esql_index_process_creation_macos
type: set_state
key: index
val:
- .ds-logs-endpoint.events.process-*
- .ds-logs-import-so-*
rule_conditions:
- type: logsource
product: macos
category: process_creation
- id: esql_index_file
type: set_state
key: index
val:
- .ds-logs-endpoint.events.file-*
- .ds-logs-windows.sysmon_operational-*
- .ds-logs-windows.forwarded-*
- .ds-logs-sysmon_linux.log-*
- .ds-logs-import-so-*
rule_cond_op: or
rule_conditions:
- type: logsource
category: file_event
- type: logsource
category: file_delete
- type: logsource
category: file_rename
- type: logsource
category: file_change
- type: logsource
category: file_access
- id: esql_index_file_windows
type: set_state
key: index
val:
- .ds-logs-endpoint.events.file-*
- .ds-logs-windows.sysmon_operational-*
- .ds-logs-windows.forwarded-*
- .ds-logs-import-so-*
rule_cond_op: or
rule_conditions:
- type: logsource
product: windows
category: file_event
- type: logsource
product: windows
category: file_delete
- type: logsource
product: windows
category: file_rename
- type: logsource
product: windows
category: file_change
- type: logsource
product: windows
category: file_access
- id: esql_index_file_linux
type: set_state
key: index
val:
- .ds-logs-endpoint.events.file-*
- .ds-logs-sysmon_linux.log-*
- .ds-logs-import-so-*
rule_cond_op: or
rule_conditions:
- type: logsource
product: linux
category: file_event
- type: logsource
product: linux
category: file_delete
- type: logsource
product: linux
category: file_rename
- type: logsource
product: linux
category: file_change
- type: logsource
product: linux
category: file_access
- id: esql_index_file_macos
type: set_state
key: index
val:
- .ds-logs-endpoint.events.file-*
- .ds-logs-import-so-*
rule_cond_op: or
rule_conditions:
- type: logsource
product: macos
category: file_event
- type: logsource
product: macos
category: file_delete
- type: logsource
product: macos
category: file_rename
- type: logsource
product: macos
category: file_change
- type: logsource
product: macos
category: file_access
- id: esql_index_registry
type: set_state
key: index
val:
- .ds-logs-endpoint.events.registry-*
- .ds-logs-windows.sysmon_operational-*
- .ds-logs-windows.forwarded-*
- .ds-logs-import-so-*
rule_cond_op: or
rule_conditions:
- type: logsource
category: registry_set
- type: logsource
category: registry_add
- type: logsource
category: registry_delete
- type: logsource
category: registry_event
- id: esql_index_library
type: set_state
key: index
val:
- .ds-logs-endpoint.events.library-*
- .ds-logs-windows.sysmon_operational-*
- .ds-logs-windows.forwarded-*
- .ds-logs-import-so-*
rule_cond_op: or
rule_conditions:
- type: logsource
category: image_load
- type: logsource
category: driver_load
- id: esql_index_endpoint_network
type: set_state
key: index
val:
- .ds-logs-endpoint.events.network-*
- .ds-logs-windows.sysmon_operational-*
- .ds-logs-windows.forwarded-*
- .ds-logs-sysmon_linux.log-*
- .ds-logs-import-so-*
rule_cond_op: or
rule_conditions:
- type: logsource
category: network_connection
- type: logsource
category: dns_query
- id: esql_index_endpoint_network_windows
type: set_state
key: index
val:
- .ds-logs-endpoint.events.network-*
- .ds-logs-windows.sysmon_operational-*
- .ds-logs-windows.forwarded-*
- .ds-logs-import-so-*
rule_cond_op: or
rule_conditions:
- type: logsource
product: windows
category: network_connection
- type: logsource
product: windows
category: dns_query
- id: esql_index_endpoint_network_linux
type: set_state
key: index
val:
- .ds-logs-endpoint.events.network-*
- .ds-logs-sysmon_linux.log-*
- .ds-logs-import-so-*
rule_cond_op: or
rule_conditions:
- type: logsource
product: linux
category: network_connection
- type: logsource
product: linux
category: dns_query
- id: esql_index_endpoint_network_macos
type: set_state
key: index
val:
- .ds-logs-endpoint.events.network-*
- .ds-logs-import-so-*
rule_cond_op: or
rule_conditions:
- type: logsource
product: macos
category: network_connection
- type: logsource
product: macos
category: dns_query
- id: esql_index_sysmon_only
type: set_state
key: index
val:
- .ds-logs-windows.sysmon_operational-*
- .ds-logs-windows.forwarded-*
- .ds-logs-import-so-*
rule_cond_op: or
rule_conditions:
- type: logsource
category: process_access
- type: logsource
category: create_remote_thread
- type: logsource
category: pipe_created
- type: logsource
category: create_stream_hash
- type: logsource
category: wmi_event
- type: logsource
category: raw_access_thread
- type: logsource
category: process_tampering
- type: logsource
category: sysmon_status
- type: logsource
category: sysmon_error
- type: logsource
category: file_executable_detected
- type: logsource
category: file_block_executable
- type: logsource
category: file_block_shredding
- type: logsource
category: clipboard_capture
- type: logsource
product: windows
service: sysmon
- id: esql_index_ps_operational
type: set_state
key: index
val:
- .ds-logs-windows.powershell_operational-*
- .ds-logs-windows.forwarded-*
- .ds-logs-import-so-*
rule_cond_op: or
rule_conditions:
- type: logsource
category: ps_script
- type: logsource
category: ps_module
- type: logsource
product: windows
service: powershell
- id: esql_index_ps_classic
type: set_state
key: index
val:
- .ds-logs-windows.powershell-*
- .ds-logs-windows.forwarded-*
- .ds-logs-import-so-*
rule_cond_op: or
rule_conditions:
- type: logsource
category: ps_classic_start
- type: logsource
category: ps_classic_provider_start
- type: logsource
category: ps_classic_script
- type: logsource
product: windows
service: powershell-classic
- id: esql_index_win_security
type: set_state
key: index
val:
- .ds-logs-system.security-*
- .ds-logs-windows.forwarded-*
- .ds-logs-import-so-*
rule_conditions:
- type: logsource
product: windows
service: security
- id: esql_index_win_system
type: set_state
key: index
val:
- .ds-logs-system.system-*
- .ds-logs-windows.forwarded-*
- .ds-logs-import-so-*
rule_conditions:
- type: logsource
product: windows
service: system
- id: esql_index_win_application
type: set_state
key: index
val:
- .ds-logs-system.application-*
- .ds-logs-windows.forwarded-*
- .ds-logs-import-so-*
rule_conditions:
- type: logsource
product: windows
service: application
- id: esql_index_linux_auth
type: set_state
key: index
val:
- .ds-logs-system.auth-*
- .ds-logs-import-so-*
rule_cond_op: or
rule_conditions:
- type: logsource
product: linux
service: auth
- type: logsource
product: linux
service: sshd
- type: logsource
product: linux
service: sudo
- id: esql_index_linux_syslog
type: set_state
key: index
val:
- .ds-logs-system.syslog-*
- .ds-logs-syslog-so-*
- .ds-logs-import-so-*
rule_conditions:
- type: logsource
product: linux
service: syslog
- id: esql_index_linux_auditd
type: set_state
key: index
val:
- .ds-logs-auditd_manager.auditd-*
- .ds-logs-auditd.log-*
- .ds-logs-import-so-*
rule_conditions:
- type: logsource
product: linux
service: auditd
- id: esql_index_network
type: set_state
key: index
val:
- .ds-logs-zeek-so-*
- .ds-logs-suricata-so-*
- .ds-logs-suricata.alerts-so-*
- .ds-logs-endpoint.events.network-*
- .ds-logs-windows.sysmon_operational-*
- .ds-logs-sysmon_linux.log-*
- .ds-logs-windows.forwarded-*
- .ds-logs-import-so-*
rule_conditions:
- type: logsource
category: network
- id: esql_index_so_network
type: set_state
key: index
val:
- .ds-logs-zeek-so-*
- .ds-logs-suricata-so-*
- .ds-logs-import-so-*
rule_cond_op: or
rule_conditions:
- type: logsource
category: network
service: connection
- type: logsource
category: network
service: dns
- type: logsource
category: network
service: http
- type: logsource
category: network
service: file
- type: logsource
category: network
service: x509
- type: logsource
category: network
service: ssl
- type: logsource
category: network
service: ssh
- type: logsource
category: dns
- id: esql_index_zeek
type: set_state
key: index
val:
- .ds-logs-zeek-so-*
- .ds-logs-import-so-*
rule_conditions:
- type: logsource
product: zeek
- id: esql_index_opencanary
type: set_state
key: index
val:
- .ds-logs-idh-so-*
- .ds-logs-import-so-*
rule_conditions:
- type: logsource
product: opencanary
- id: esql_index_kratos
type: set_state
key: index
val:
- .ds-logs-kratos-so-*
- .ds-logs-import-so-*
rule_conditions:
- type: logsource
product: kratos
# SOC reads the mapped group-by columns from this output
postprocessing:
- id: esql_correlation_group_by
type: template
template: '{{ {"query": query, "group_by": rule.group_by} | tojson }}'
rule_conditions:
- type: is_sigma_correlation_rule
@@ -2,13 +2,13 @@ name: Security Onion - Playbook Pipeline
priority: 97
transformations:
# Route to lowercase-normalized .caseless subfields for case-insensitive matching.
# file.path.caseless exists on Defend only (Sysmon file events lack it);
# registry.path / dll.path / file.name have no .caseless on any source.
- id: case_insensitive_string_fields
type: field_name_mapping
mapping:
process.executable: process.executable.caseless
process.parent.executable: process.parent.executable.caseless
process.parent.name: process.parent.name.caseless
process.command_line: process.command_line.caseless
process.parent.command_line: process.parent.command_line.caseless
file.path: file.path.caseless
@@ -1,38 +1,6 @@
name: Security Onion Baseline Pipeline
priority: 90
transformations:
# ES|QL scalar == returns null on multivalued fields; the
# backend reads this key and emits MV_INTERSECTS instead.
- id: declare_multivalue_fields
type: set_state
key: multivalue_fields
val:
- event.type
- event.action
- event.category
- tags
- process.args
- related.ip
- dns.resolved_ip
# always lowercase: matched exactly with the indexed ':' operator
- id: case_sensitive_categorization_fields
type: set_state
key: case_insensitive_exempt_fields
val:
- tags
- event.category
- event.type
- event.kind
# Not every source maps .caseless; EQL/ES|QL already match case-insensitively.
- id: caseless_to_parent_fields
type: field_name_mapping
mapping:
process.executable.caseless: process.executable
process.name.caseless: process.name
process.parent.executable.caseless: process.parent.executable
process.parent.name.caseless: process.parent.name
target.process.executable.caseless: target.process.executable
target.process.name.caseless: target.process.name
- id: baseline_field_name_mapping
type: field_name_mapping
mapping:
@@ -127,9 +95,6 @@ transformations:
valid_hash_algos: ["MD5", "SHA1", "SHA256", "SHA512", "IMPHASH"]
field_prefix: "file"
drop_algo_prefix: False
# ecs_windows renamed Hashes; pySigma 1.5+ parses only these
field_to_parse:
- winlog.event_data.Hashes
field_name_conditions:
- type: include_fields
fields:
-30
View File
@@ -8,7 +8,6 @@
{% from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_NODES %}
{% from 'manager/map.jinja' import MANAGERMERGED %}
{% from 'telegraf/map.jinja' import TELEGRAFMERGED %}
{% from 'elastalert/map.jinja' import ELASTALERTMERGED %}
{%- set PG_ENTRY = salt['pillar.get']('telegraf:postgres_creds:' ~ grains.id, {}) %}
{%- set PG_USER = PG_ENTRY.get('user', '') %}
{%- set PG_PASS = PG_ENTRY.get('pass', '') %}
@@ -22,25 +21,6 @@
{% do SOCMERGED.config.server.update({'additionalCA': MANAGERMERGED.additionalCA}) %}
{% do SOCMERGED.config.server.update({'insecureSkipVerify': MANAGERMERGED.insecureSkipVerify}) %}
{# pillar.get replaces lists instead of merging them, so combine availableModels by id #}
{% set default_models = {} %}
{% for m in SOCDEFAULTS.config.server.client.assistant.availableModels %}
{% do default_models.setdefault(m.id, m) %}
{% endfor %}
{% set merged_models = [] %}
{% set merged_ids = [] %}
{% for m in SOCMERGED.config.server.client.assistant.availableModels or [] %}
{% set model = {} %}
{% do model.update(default_models.get(m.get('id'), {})) %}
{% do model.update(m) %}
{% do merged_models.append(model) %}
{% do merged_ids.append(m.get('id')) %}
{% endfor %}
{% for id, m in default_models.items() if id not in merged_ids %}
{% do merged_models.append(m) %}
{% endfor %}
{% do SOCMERGED.config.server.client.assistant.update({'availableModels': merged_models}) %}
{% if not SOCMERGED.config.server.modules.postgres.host %}
{% do SOCMERGED.config.server.modules.postgres.update({'host': GLOBALS.manager}) %}
{% endif %}
@@ -83,10 +63,6 @@
{% do SOCMERGED.config.server.modules.elastalertengine.update({'enabledSigmaRules': SOCMERGED.config.server.modules.elastalertengine.enabledSigmaRules.default}) %}
{% endif %}
{# correlation schedules follow ElastAlert's run_every #}
{% set run_every = ELASTALERTMERGED.config.run_every %}
{% do SOCMERGED.config.server.modules.elastalertengine.update({'elastAlertRunEverySeconds': run_every.get('minutes', 0) * 60 + run_every.get('seconds', 0)}) %}
{# set elastalertengine.rulesRepos, strelkaengine.rulesRepos, and suricataengine.rulesetSources based on airgap or not #}
{% if GLOBALS.airgap %}
{% do SOCMERGED.config.server.modules.elastalertengine.update({'rulesRepos': SOCMERGED.config.server.modules.elastalertengine.rulesRepos.airgap}) %}
@@ -104,12 +80,6 @@
{% do SOCMERGED.config.server.update({'airgapEnabled': false}) %}
{% endif %}
{# correlation authoring requires ES|QL #}
{% if not SOCMERGED.config.server.modules.elastalertengine.useEsql %}
{% do SOCMERGED.config.server.client.detection.templateDetections.pop('elastalert_correlation', None) %}
{% do SOCMERGED.config.server.client.detections.update({'queries': SOCMERGED.config.server.client.detections.queries | rejectattr('name', 'equalto', 'Detection Type - Sigma (Elastalert) - Correlations') | list}) %}
{% endif %}
{# Define the postgresmetrics module if telegraf is setup to only use Postgres #}
{% if TELEGRAFMERGED.output != 'INFLUXDB' and PG_USER and PG_PASS %}
{% do SOCMERGED.config.server.modules.update({
+1 -161
View File
@@ -155,15 +155,6 @@ soc:
description: Path to custom markdown templates for PDF report generation. All markdown files in this directory will be available as custom reports in the SOC Reports interface.
global: True
advanced: True
schedules:
title: Schedules
description: Schedules that are shared across the Security Onion product. Modify via one of the SOC Schedules view.
readonlyUi: True
global: True
advanced: True
forcedType: string
syntax: json
storage: db
subgrids:
title: Subordinate Grids
description: |
@@ -401,27 +392,10 @@ soc:
advanced: False
helpLink: sigma
useEsql:
description: "(Pre-release) Use Elasticsearch Piped Query Language (ES|QL) instead of EQL (Elastic Query Language) for Elasticsearch queries. The Sigma converter will output ES|QL instead of EQL, allowing support for correlations. Switching back to EQL is not supported for correlations."
description: "(Pre-release) Use Elasticsearch Piped Query Language (ES|QL) instead of EQL (Elastic Query Language) for Elasticsearch queries. The Sigma converter will output ES|QL instead of EQL, allowing support for correlations."
global: True
advanced: True
forcedType: bool
esqlCaseInsensitive:
description: "Match string values case-insensitively when converting Sigma rules, and group correlation values regardless of case. Applies to ES|QL only"
global: True
advanced: True
forcedType: bool
esqlQueryDelaySeconds:
description: "Seconds ES|QL rules search behind now, so unsearchable events aren't missed. Delays alerts by the same amount. Set at least the longest index refresh interval. ES|QL only."
global: True
advanced: True
forcedType: int
helpLink: sigma
esqlCorrelationAllowanceSeconds:
description: "Extra seconds of arrivals each correlation run re-reads beyond its timespan, so a burst whose events arrive spread out is still counted together. Correlations below a threshold (lt, lte, eq, neq) and value_avg or value_percentile correlations count only the timespan ending this much plus esqlQueryDelaySeconds ago, so they alert this much later. ES|QL only."
global: True
advanced: True
forcedType: int
helpLink: sigma
elastic:
index:
description: Comma-separated list of indices or index patterns (wildcard "*" supported) that SOC will search for records.
@@ -502,33 +476,6 @@ soc:
global: True
advanced: True
forcedType: bool
notification:
destinations:
title: Notification Destinations
description: JSON list of notifications. Modify via the SOC Notifications view.
readonlyUi: True
global: True
advanced: True
forcedType: string
syntax: json
storage: db
dismissedPruneDays:
title: Dismissed Retention Days
description: The number of days to retain dismissed notifications. When a notification is dismissed, it will be pruned after this many days. Only one user need dismiss a notification for it to be pruned.
forcedType: int
global: True
maxListLimit:
description: Maximum number of notifications to display.
forcedType: int
global: True
enabled:
description: Enables or disables the SOC notification module.
forcedType: bool
global: True
connectionTimeoutSeconds:
description: Duration (in seconds) to wait for a response from the remote notification endpoint host before giving up.
forcedType: int
global: True
postgres:
host:
description: Hostname or IP address of the PostgreSQL server used by SOC. Defaults to the manager hostname.
@@ -555,48 +502,6 @@ soc:
global: True
sensitive: True
advanced: True
postgresmetrics:
host:
description: Hostname or IP address of the PostgreSQL server used by Telegraf. Defaults to the manager hostname.
global: True
advanced: True
port:
description: Port of the PostgreSQL server used by Telegraf.
global: True
advanced: True
sslMode:
description: "Use encrypted connections to the PostgreSQL server used by Telegraf. Must be one of the following values: disable, allow, prefer, require, verify-ca, verify-full."
global: True
advanced: True
database:
description: Database to authenticate to on the PostgreSQL server.
global: True
advanced: True
user:
description: Username to authenticate to the PostgreSQL server used by Telegraf.
global: True
advanced: True
password:
description: Password used to authenticate to the PostgreSQL server used by Telegraf.
global: True
sensitive: True
advanced: True
cacheExpirationMs:
description: The interval (in milliseconds) to wait before querying the DB for updated metrics.
global: True
advanced: True
maxMetricAgeSeconds:
description: The maximum age (in seconds) of metrics to display in the SOC Grid Metrics view. Metrics older than this value will not be displayed.
global: True
advanced: True
alarms:
description: JSON list of metric alarms. Modify via the SOC Grid Alarms view.
readonlyUi: True
advanced: True
global: True
forcedType: string
syntax: json
storage: db
salt:
longRelayTimeoutMs:
description: Duration (in milliseconds) to wait for a response from the Salt API when executing tasks known for being long running before giving up and showing an error on the SOC UI.
@@ -855,7 +760,6 @@ soc:
- gemini
- openai_responses
- openai_chat
- openai_embeddings
- field: apiUrl
label: API URL
required: False
@@ -877,15 +781,6 @@ soc:
description: Indicates if the Assistant Module should operate in agentic mode or not. If true, agents can work together to solve tasks.
global: True
forcedType: bool
automations:
description: Scheduled automations for the Onion AI assistant, managed from the Agent Studio.
global: True
advanced: True
readonlyUi: True
storage: db
forcedType: string
syntax: json
helpLink: onion-ai
agents:
description: Agent definitions for the Onion AI assistant, managed from the Agent Studio. An entry naming a system agent overrides only the fields an admin may change; everything else comes from the built-in definition.
global: True
@@ -918,9 +813,6 @@ soc:
- field: persona
label: Persona
multiline: True
- field: maxConcurrentInstances
label: Max Concurrent Instances
forcedType: int
skills:
description: Skill definitions for the Onion AI assistant, managed from the Agent Studio. An entry naming a system skill overrides only its enabled state and persona addendum; its tool set comes from the built-in definition.
global: True
@@ -953,12 +845,6 @@ soc:
DetectionEngineer:
description: This agent manages detections and their overrides, including tuning noisy rules and authoring rule content.
global: True
AlertTriage:
description: This agent triages alerts autonomously for the Alert Triage automation, ending each run with a report and an assessment of the alert. It can notify through the Notifier but cannot acknowledge alerts or escalate to cases.
global: True
Notifier:
description: This agent sends a single notification on behalf of another agent, such as AlertTriage, and takes no other action.
global: True
useMemory:
description: Enables the Memory system for OnionAI
global: True
@@ -1030,48 +916,6 @@ soc:
description: The number of times to retry extracting memories from a session if errors occur.
global: True
advanced: True
agentSessionMaxTurns:
description: Maximum number of model turns a headless agent session, such as one started by an automation, may take before it is stopped. Turns taken by delegated sub-agents count toward this limit. A session that reaches the limit is recorded as failed.
global: True
advanced: True
forcedType: int
agentStreamFlushIntervalMs:
description: Milliseconds between writes of a streaming headless agent turn to the database. Lower values show progress sooner in the Agent Studio at the cost of more frequent Elasticsearch updates.
global: True
advanced: True
forcedType: int
agentStreamIdleTimeoutSeconds:
description: Seconds a streaming headless agent turn may go without receiving any output before it is abandoned and the session is recorded as failed. Set to 0 to disable the timeout.
global: True
advanced: True
forcedType: int
automationSettings:
tickIntervalSeconds:
description: How often, in seconds, the automation scheduler checks for automations that are due to run. Must be greater than 0. This value is also the default interval for new automations, however admins can override individual automation intervals to a longer value via the Agent Studio.
global: True
advanced: True
forcedType: int
maxConcurrentItems:
description: Maximum number of automation work items that may run at the same time. Additional work items wait in the queue until a running item finishes. User chat sessions count toward this limit but are never held back by it. Set to 0 to disable the limit.
global: True
advanced: True
forcedType: int
maxQueuedItems:
description: Maximum number of automation work items that may wait to start. Once the queue is full, no new work items are created until the backlog drains. Set to 0 to disable the limit.
global: True
advanced: True
forcedType: int
alertTriageEpoch:
description: The earliest alert time the Alert Triage automation will consider. Alerts before this time are never triaged, which keeps a first run on an existing deployment from working through old history. Must be in UTC format (2026-09-24T00:00:00Z).
regex: '^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?Z)?$'
regexFailureMessage: Expecting date in RFC3339 format (2026-09-24T00:00:00Z)
global: True
advanced: True
tools:
filterEventFields:
description: A whitelist of fields to return when OnionAI uses the query_events tool. All other fields are removed. One field per line.
global: True
multiline: True
client:
assistant:
enabled:
@@ -1173,10 +1017,6 @@ soc:
description: Set to true to enable the Detections module in SOC.
global: True
forcedType: bool
allowExternalMarkdownImages:
description: Set to true to let user-written Markdown, such as case descriptions and comments, load images from other servers. Loading an image sends a request to its server, so leave this disabled unless needed; Onion AI output never loads external images.
global: True
forcedType: bool
inactiveTools:
description: List of external tools to remove from the SOC UI.
global: True
+4 -4
View File
@@ -1333,8 +1333,8 @@ DISA STIG for Oracle Linux 9 V1R3.</xccdf-1.2:description>
<xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_dir_group_ownership_library_dirs" selected="true"/>
<xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_dir_ownership_library_dirs" selected="true"/>
<xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_dir_permissions_library_dirs" selected="true"/>
<xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_dir_perms_world_writable_root_owned" selected="false"/>
<xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_dir_perms_world_writable_sticky_bits" selected="false"/>
<xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_dir_perms_world_writable_root_owned" selected="true"/>
<xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_dir_perms_world_writable_sticky_bits" selected="true"/>
<xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_directory_group_ownership_var_log_audit" selected="true"/>
<xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_directory_ownership_var_log_audit" selected="true"/>
<xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_disable_ctrlaltdel_burstaction" selected="true"/>
@@ -1935,8 +1935,8 @@ standard DISA STIG for Oracle Linux 9 profile.</xccdf-1.2:description>
<xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_dir_group_ownership_library_dirs" selected="true"/>
<xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_dir_ownership_library_dirs" selected="true"/>
<xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_dir_permissions_library_dirs" selected="true"/>
<xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_dir_perms_world_writable_root_owned" selected="false"/>
<xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_dir_perms_world_writable_sticky_bits" selected="false"/>
<xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_dir_perms_world_writable_root_owned" selected="true"/>
<xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_dir_perms_world_writable_sticky_bits" selected="true"/>
<xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_directory_group_ownership_var_log_audit" selected="true"/>
<xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_directory_ownership_var_log_audit" selected="true"/>
<xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_disable_ctrlaltdel_burstaction" selected="true"/>
+2 -2
View File
@@ -51,8 +51,8 @@ strelka_sbin:
file.recurse:
- name: /usr/sbin
- source: salt://strelka/tools/sbin
- user: root
- group: root
- user: 939
- group: 939
- file_mode: 755
{% else %}
+4 -5
View File
@@ -64,7 +64,6 @@ suricata:
- gid: 940
- home: /nsm/suricata
- createhome: False
- shell: /sbin/nologin
socoregroupwithsuricata:
group.present:
@@ -77,16 +76,16 @@ suricata_sbin:
file.recurse:
- name: /usr/sbin
- source: salt://suricata/tools/sbin
- user: root
- group: root
- user: 939
- group: 939
- file_mode: 755
suricata_sbin_jinja:
file.recurse:
- name: /usr/sbin
- source: salt://suricata/tools/sbin_jinja
- user: root
- group: root
- user: 939
- group: 939
- file_mode: 755
- template: jinja
Loaded 100 of 119 files, more files were not shown because too many files have changed in this diff. Show more