Compare commits

..

1 Commits

Author SHA1 Message Date
Mike Reeves 213afe4875 oracle 10 testing 2026-06-03 14:11:30 -04:00
39 changed files with 154 additions and 624 deletions
View File
-1
View File
@@ -11,7 +11,6 @@ body:
- -
- 3.0.0 - 3.0.0
- 3.1.0 - 3.1.0
- 3.2.0
- Other (please provide detail below) - Other (please provide detail below)
validations: validations:
required: true required: true
+1 -1
View File
@@ -1 +1 @@
20260528
+1 -1
View File
@@ -1 +1 @@
3.2.0 3.1.0
@@ -25,11 +25,9 @@ if [ ! -f $BACKUPFILE ]; then
# Create empty backup file # Create empty backup file
tar -cf $BACKUPFILE -T /dev/null tar -cf $BACKUPFILE -T /dev/null
# Loop through all paths defined in global.sls, and append them to backup file if they exist # Loop through all paths defined in global.sls, and append them to backup file
{%- for LOCATION in BACKUPLOCATIONS %} {%- for LOCATION in BACKUPLOCATIONS %}
if [[ -d {{ LOCATION }} || -f {{ LOCATION }} ]]; then
tar -rf $BACKUPFILE "${EXCLUSIONS[@]}" {{ LOCATION }} tar -rf $BACKUPFILE "${EXCLUSIONS[@]}" {{ LOCATION }}
fi
{%- endfor %} {%- endfor %}
fi fi
-11
View File
@@ -130,17 +130,6 @@ common_sbin:
- so-pcap-import - so-pcap-import
{% endif %} {% endif %}
# Pin physical NIC names by MAC (run-once) so a kernel upgrade can't renumber the
# interfaces SO binds by name. The marker keeps it a one-time setup; an admin can
# pre-create the marker to opt out.
pin_nic_names:
cmd.run:
- name: /usr/sbin/so-nic-pin
- unless: 'test -e /opt/so/state/nic_names_pinned'
- require:
- file: common_sbin
- file: statedir
common_sbin_jinja: common_sbin_jinja:
file.recurse: file.recurse:
- name: /usr/sbin - name: /usr/sbin
+20 -1
View File
@@ -26,14 +26,33 @@ commonpkgs:
- net-tools - net-tools
- nmap-ncat - nmap-ncat
- procps-ng - procps-ng
{# OL10 test path: python3-docker / python3-m2crypto are not packaged in EPEL 10 and are not
referenced by SO code (salt uses its bundled docker module from salt/python_modules.sls).
python3-rich is also unavailable on EL10 (its pygments dep is not packaged), so it is
installed via pip below. Gate on the grain because GLOBALS/pillars are not available this
early (see header note). #}
{% if grains['osmajorrelease']|int < 10 %}
- python3-docker - python3-docker
- python3-m2crypto - python3-m2crypto
- python3-rich
{% else %}
- python3-pip
{% endif %}
- python3-packaging - python3-packaging
- python3-pyyaml - python3-pyyaml
- python3-rich
- rsync - rsync
- sqlite - sqlite
- tcpdump - tcpdump
- unzip - unzip
- wget - wget
- yum-utils - yum-utils
{% if grains['osmajorrelease']|int >= 10 %}
# OL10 test path: rich is not packaged for EL10; install it into the system python3 for so-status.
commonpkgs_pip_rich:
cmd.run:
- name: python3 -m pip install rich
- unless: python3 -c "import rich"
- require:
- pkg: commonpkgs
{% endif %}
+7 -2
View File
@@ -354,7 +354,12 @@ gpg_rpm_import() {
else else
local RPMKEYSLOC="$UPDATE_DIR/salt/repo/client/files/$OS/keys" local RPMKEYSLOC="$UPDATE_DIR/salt/repo/client/files/$OS/keys"
fi fi
if [[ "$OSVER" == "10" ]]; then
# OL10 test path uses public repos; the public oracle-epel-release and docker repos provide their own keys
RPMKEYS=('RPM-GPG-KEY-oracle' 'SALT-PROJECT-GPG-PUBKEY-2023.pub')
else
RPMKEYS=('RPM-GPG-KEY-oracle' 'RPM-GPG-KEY-EPEL-9' 'SALT-PROJECT-GPG-PUBKEY-2023.pub' 'docker.pub' 'securityonion.pub') RPMKEYS=('RPM-GPG-KEY-oracle' 'RPM-GPG-KEY-EPEL-9' 'SALT-PROJECT-GPG-PUBKEY-2023.pub' 'docker.pub' 'securityonion.pub')
fi
for RPMKEY in "${RPMKEYS[@]}"; do for RPMKEY in "${RPMKEYS[@]}"; do
rpm --import $RPMKEYSLOC/$RPMKEY rpm --import $RPMKEYSLOC/$RPMKEY
echo "Imported $RPMKEY" echo "Imported $RPMKEY"
@@ -626,9 +631,9 @@ salt_minion_count() {
} }
set_os() { set_os() {
if [ -f /etc/redhat-release ] && grep -q "Red Hat Enterprise Linux release 9" /etc/redhat-release && [ -f /etc/oracle-release ]; then if [ -f /etc/oracle-release ] && grep -qE "release (9|10)\b" /etc/oracle-release; then
OS=oracle OS=oracle
OSVER=9 OSVER=$(grep -oE "release [0-9]+" /etc/oracle-release | grep -oE "[0-9]+")
is_oracle=true is_oracle=true
is_rpm=true is_rpm=true
fi fi
+15
View File
@@ -115,6 +115,21 @@ update_docker_containers() {
rm -rf $SIGNPATH >> "$LOG_FILE" 2>&1 rm -rf $SIGNPATH >> "$LOG_FILE" 2>&1
mkdir -p $SIGNPATH >> "$LOG_FILE" 2>&1 mkdir -p $SIGNPATH >> "$LOG_FILE" 2>&1
# OL10 test path: GnuPG 2.4 enables the keybox daemon (keyboxd) by default, which deadlocks
# under the rapid sequential gpg --verify calls below ("waiting for lock ... keydb_search
# failed: Connection timed out ... No public key"). Editing the default homedir's common.conf
# is unreliable (gpg re-adds use-keyboxd when it re-initializes the homedir), so run all the
# image-signature gpg ops in a dedicated homedir whose pre-written common.conf leaves keyboxd
# off, forcing the classic keybox. Isolated from the system keyring and deterministic.
if [ "$OSVER" = "10" ]; then
export GNUPGHOME="$SIGNPATH/gnupg"
rm -rf "$GNUPGHOME" >> "$LOG_FILE" 2>&1
mkdir -p "$GNUPGHOME" >> "$LOG_FILE" 2>&1
chmod 700 "$GNUPGHOME"
echo "# keyboxd disabled for SO image signature verification on EL10" > "$GNUPGHOME/common.conf"
gpgconf --kill keyboxd gpg-agent >> "$LOG_FILE" 2>&1 || true
fi
# Let's make sure we have the public key # Let's make sure we have the public key
run_check_net_err \ run_check_net_err \
"curl --retry 5 --retry-delay 60 -sSL https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS -o $SIGNPATH/KEYS" \ "curl --retry 5 --retry-delay 60 -sSL https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS -o $SIGNPATH/KEYS" \
-76
View File
@@ -1,76 +0,0 @@
#!/bin/bash
#
# so-nic-pin — pin physical NIC names by permanent MAC via classic by-MAC udev
# rules, so a kernel upgrade can't renumber them.
#
# Security Onion binds its management and monitor interfaces BY NAME in pillar
# (host:mainint, sensor:mainint, and bond0 is built on a specific physical NIC).
# A kernel upgrade can change the kernel/systemd-udevd predictable-naming output
# and renumber those NICs (e.g. enp1s0 -> enp2s0), which breaks the grid: the
# pillar references a name that no longer exists and bond/bridge bring-up fails.
#
# This writes /etc/udev/rules.d/70-persistent-net.rules pinning each PHYSICAL NIC
# to its CURRENT name by its PERMANENT MAC, freezing the names across future kernel
# changes. It only writes the rules file; it does NOT live-trigger a rename (the
# rules apply on the next boot/kernel, and a live rename would be disruptive).
#
# Run-once: gated by the drop file /opt/so/state/nic_names_pinned. If the marker is
# present the script does nothing, so an admin can pre-create it to opt out. Invoked
# from the common state on every highstate; the marker keeps it a one-time setup.
NET_RULES_FILE="/etc/udev/rules.d/70-persistent-net.rules"
MARKER="/opt/so/state/nic_names_pinned"
log() { echo -e "[so-nic-pin] $*"; }
# Echo "<name> <permanent-mac>" for every PHYSICAL NIC. A physical NIC is backed by a
# real device (has device/driver), which excludes bond0/sobridge/docker0/veth*/lo whose
# MACs are dynamic and must never be pinned. The PERMANENT MAC is used (ethtool -P, with
# fallbacks), not the current one: an enslaved bond member's current MAC is rewritten to
# the bond's, so matching on it would be wrong/ambiguous.
physical_nics() {
local path n mac
for path in /sys/class/net/*; do
n="${path##*/}"
[ "$n" = "lo" ] && continue
[ -e "${path}/device/driver" ] || continue # real device only
mac="$(ethtool -P "$n" 2>/dev/null | awk '/Permanent address/{print $NF}')"
case "$mac" in ""|00:00:00:00:00:00) mac="$(cat "${path}/bonding_slave/perm_hwaddr" 2>/dev/null)" ;; esac
case "$mac" in ""|00:00:00:00:00:00) mac="$(cat "${path}/address" 2>/dev/null)" ;; esac
case "$mac" in ""|00:00:00:00:00:00) continue ;; esac
echo "$n $mac"
done
}
# Turn "<name> <mac>" lines on stdin into classic by-MAC persistent-net udev rules.
render_net_rules() {
echo "# Generated by so-nic-pin: pin NIC names by MAC so kernel upgrades can't renumber them."
echo "# Security Onion binds its management/monitor interfaces by name; do not hand-edit."
local n mac
while read -r n mac; do
[ -n "$n" ] || continue
printf 'SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ATTR{address}=="%s", NAME="%s"\n' \
"$mac" "$n"
done
}
[ "$(id -u)" -eq 0 ] || exit 0 # salt runs us as root; bail quietly otherwise
[ -e "${MARKER}" ] && exit 0 # run-once guard (mirrors the state's unless)
nics="$(physical_nics)"
if [ -z "${nics}" ]; then
log "no physical NICs detected — nothing to pin (will retry on next highstate)"
exit 0 # do NOT drop the marker; let it retry later
fi
log "pinning physical NICs by permanent MAC:"
echo "${nics}" | sed 's/^/ /'
[ -f "${NET_RULES_FILE}" ] && cp -f "${NET_RULES_FILE}" "${NET_RULES_FILE}.bak"
echo "${nics}" | render_net_rules > "${NET_RULES_FILE}" || {
log "ERROR: failed to write ${NET_RULES_FILE}"
exit 1
}
mkdir -p "$(dirname "${MARKER}")" && touch "${MARKER}"
log "wrote ${NET_RULES_FILE} ($(grep -c '^SUBSYSTEM' "${NET_RULES_FILE}") NIC(s) pinned); dropped ${MARKER}"
+8
View File
@@ -18,10 +18,18 @@ dockergroup:
dockerheldpackages: dockerheldpackages:
pkg.installed: pkg.installed:
- pkgs: - pkgs:
{% if GLOBALS.os_version|int >= 10 %}
# OL10 test path: install latest Docker CE from the public repo (no .el9 builds available)
- containerd.io
- docker-ce
- docker-ce-cli
- docker-ce-rootless-extras
{% else %}
- containerd.io: 2.2.1-1.el9 - containerd.io: 2.2.1-1.el9
- docker-ce: 3:29.2.1-1.el9 - docker-ce: 3:29.2.1-1.el9
- docker-ce-cli: 1:29.2.1-1.el9 - docker-ce-cli: 1:29.2.1-1.el9
- docker-ce-rootless-extras: 29.2.1-1.el9 - docker-ce-rootless-extras: 29.2.1-1.el9
{% endif %}
- hold: True - hold: True
- update_holds: True - update_holds: True
-11
View File
@@ -101,17 +101,6 @@ so-elastic-fleet:
- file: trusttheca - file: trusttheca
- x509: etc_elasticfleet_key - x509: etc_elasticfleet_key
- x509: etc_elasticfleet_crt - x509: etc_elasticfleet_crt
wait_for_so-elastic-fleet:
http.wait_for_successful_query:
- name: "https://localhost:8220/api/status"
- ssl: True
- verify_ssl: False
- status: 200
- wait_for: 300
- request_interval: 15
- require:
- docker_container: so-elastic-fleet
{% endif %} {% endif %}
delete_so-elastic-fleet_so-status.disabled: delete_so-elastic-fleet_so-status.disabled:
+2 -25
View File
@@ -9,19 +9,15 @@
include: include:
- elasticfleet.config - elasticfleet.config
- kibana.enabled
# If enabled, automatically update Fleet Logstash Outputs # If enabled, automatically update Fleet Logstash Outputs
{% if ELASTICFLEETMERGED.config.server.enable_auto_configuration %} {% if ELASTICFLEETMERGED.config.server.enable_auto_configuration and grains.role not in ['so-import', 'so-eval'] %}
{% if grains.role not in ['so-import', 'so-eval']%}
so-elastic-fleet-auto-configure-logstash-outputs: so-elastic-fleet-auto-configure-logstash-outputs:
cmd.run: cmd.run:
- name: /usr/sbin/so-elastic-fleet-outputs-update - name: /usr/sbin/so-elastic-fleet-outputs-update
- retry: - retry:
attempts: 4 attempts: 4
interval: 30 interval: 30
- require:
- http: wait_for_so-kibana
{% endif %} {% endif %}
# If enabled, automatically update Fleet Server URLs & ES Connection # If enabled, automatically update Fleet Server URLs & ES Connection
@@ -31,9 +27,6 @@ so-elastic-fleet-auto-configure-server-urls:
- retry: - retry:
attempts: 4 attempts: 4
interval: 30 interval: 30
- require:
- http: wait_for_so-kibana
{% endif %}
# Automatically update Fleet Server Elasticsearch URLs & Agent Artifact URLs # Automatically update Fleet Server Elasticsearch URLs & Agent Artifact URLs
so-elastic-fleet-auto-configure-elasticsearch-urls: so-elastic-fleet-auto-configure-elasticsearch-urls:
@@ -42,8 +35,6 @@ so-elastic-fleet-auto-configure-elasticsearch-urls:
- retry: - retry:
attempts: 4 attempts: 4
interval: 30 interval: 30
- require:
- http: wait_for_so-kibana
so-elastic-fleet-auto-configure-artifact-urls: so-elastic-fleet-auto-configure-artifact-urls:
cmd.run: cmd.run:
@@ -51,8 +42,6 @@ so-elastic-fleet-auto-configure-artifact-urls:
- retry: - retry:
attempts: 4 attempts: 4
interval: 30 interval: 30
- require:
- http: wait_for_so-kibana
so-elastic-fleet-package-statefile: so-elastic-fleet-package-statefile:
file.managed: file.managed:
@@ -64,9 +53,7 @@ so-elastic-fleet-package-upgrade:
- name: /usr/sbin/so-elastic-fleet-package-upgrade - name: /usr/sbin/so-elastic-fleet-package-upgrade
- retry: - retry:
attempts: 3 attempts: 3
interval: 30 interval: 10
- require:
- http: wait_for_so-kibana
- onchanges: - onchanges:
- file: /opt/so/state/elastic_fleet_packages.txt - file: /opt/so/state/elastic_fleet_packages.txt
@@ -76,8 +63,6 @@ so-elastic-fleet-integrations:
- retry: - retry:
attempts: 3 attempts: 3
interval: 10 interval: 10
- require:
- http: wait_for_so-kibana
so-elastic-agent-grid-upgrade: so-elastic-agent-grid-upgrade:
cmd.run: cmd.run:
@@ -85,8 +70,6 @@ so-elastic-agent-grid-upgrade:
- retry: - retry:
attempts: 12 attempts: 12
interval: 5 interval: 5
- require:
- http: wait_for_so-kibana
so-elastic-fleet-integration-upgrade: so-elastic-fleet-integration-upgrade:
cmd.run: cmd.run:
@@ -94,22 +77,16 @@ so-elastic-fleet-integration-upgrade:
- retry: - retry:
attempts: 3 attempts: 3
interval: 10 interval: 10
- require:
- http: wait_for_so-kibana
{# Optional integrations script doesn't need the retries like so-elastic-fleet-integration-upgrade which loads the default integrations #} {# Optional integrations script doesn't need the retries like so-elastic-fleet-integration-upgrade which loads the default integrations #}
so-elastic-fleet-addon-integrations: so-elastic-fleet-addon-integrations:
cmd.run: cmd.run:
- name: /usr/sbin/so-elastic-fleet-optional-integrations-load - name: /usr/sbin/so-elastic-fleet-optional-integrations-load
- require:
- http: wait_for_so-kibana
{% if ELASTICFLEETMERGED.config.defend_filters.enable_auto_configuration %} {% if ELASTICFLEETMERGED.config.defend_filters.enable_auto_configuration %}
so-elastic-defend-manage-filters-file-watch: so-elastic-defend-manage-filters-file-watch:
cmd.run: cmd.run:
- name: python3 /sbin/so-elastic-defend-manage-filters.py -c /opt/so/conf/elasticsearch/curl.config -d /opt/so/conf/elastic-fleet/defend-exclusions/disabled-filters.yaml -i /nsm/securityonion-resources/event_filters/ -i /opt/so/conf/elastic-fleet/defend-exclusions/rulesets/custom-filters/ &>> /opt/so/log/elasticfleet/elastic-defend-manage-filters.log - name: python3 /sbin/so-elastic-defend-manage-filters.py -c /opt/so/conf/elasticsearch/curl.config -d /opt/so/conf/elastic-fleet/defend-exclusions/disabled-filters.yaml -i /nsm/securityonion-resources/event_filters/ -i /opt/so/conf/elastic-fleet/defend-exclusions/rulesets/custom-filters/ &>> /opt/so/log/elasticfleet/elastic-defend-manage-filters.log
- require:
- http: wait_for_so-kibana
- onchanges: - onchanges:
- file: elasticdefendcustom - file: elasticdefendcustom
- file: elasticdefenddisabled - file: elasticdefenddisabled
@@ -108,12 +108,9 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then
done done
# Only create the state file if all policies were created/updated successfully # Only create the state file if all policies were created/updated successfully
if [[ $RETURN_CODE -eq 0 ]]; then if [[ "$RETURN_CODE" != "1" ]]; then
touch /opt/so/state/eaintegrations.txt touch /opt/so/state/eaintegrations.txt
else
exit 1
fi fi
else else
echo "Fleet integration policies already loaded." exit $RETURN_CODE
exit 0
fi fi
@@ -8,33 +8,18 @@
. /usr/sbin/so-elastic-fleet-common . /usr/sbin/so-elastic-fleet-common
PKG_LOAD_FAILURES=0
PKG_LOAD_FAILURES_NAMES=()
{%- for PACKAGE in SUPPORTED_PACKAGES %} {%- for PACKAGE in SUPPORTED_PACKAGES %}
echo "Upgrading {{ PACKAGE }} package..." echo "Upgrading {{ PACKAGE }} package..."
if VERSION=$(elastic_fleet_package_latest_version_check "{{ PACKAGE }}"); then if VERSION=$(elastic_fleet_package_latest_version_check "{{ PACKAGE }}"); then
if ! elastic_fleet_package_install "{{ PACKAGE }}" "$VERSION"; then if ! elastic_fleet_package_install "{{ PACKAGE }}" "$VERSION"; then
PKG_LOAD_FAILURES=$((PKG_LOAD_FAILURES + 1)) # exit 1 on failure to upgrade a default package, allow salt to handle retries
PKG_LOAD_FAILURES_NAMES+=("{{ PACKAGE }}") echo -e "\nERROR: Failed to upgrade $PACKAGE to version: $VERSION"
exit 1
fi fi
else else
PKG_LOAD_FAILURES=$((PKG_LOAD_FAILURES + 1)) echo -e "\nERROR: Failed to get version information for integration $PACKAGE"
PKG_LOAD_FAILURES_NAMES+=("{{ PACKAGE }}")
fi fi
echo echo
{%- endfor %} {%- endfor %}
if [ $PKG_LOAD_FAILURES -gt 0 ]; then
echo "ERROR: Failed to upgrade $PKG_LOAD_FAILURES package(s):"
for PKG in "${PKG_LOAD_FAILURES_NAMES[@]}"; do
echo " - $PKG"
done
# exit 1 on failure to upgrade a default package, allow salt to handle retries
exit 1
else
echo "Successfully upgraded all packages."
fi
echo echo
/usr/sbin/so-elasticsearch-templates-load /usr/sbin/so-elasticsearch-templates-load
+1 -18
View File
@@ -9,12 +9,9 @@
{% from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %} {% from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
{% from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS, SO_MANAGED_INDICES %} {% from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS, SO_MANAGED_INDICES %}
{% if GLOBALS.role != 'so-heavynode' %} {% if GLOBALS.role != 'so-heavynode' %}
{% from 'elasticsearch/template.map.jinja' import ALL_ADDON_SETTINGS, ADDON_INDICES %} {% from 'elasticsearch/template.map.jinja' import ALL_ADDON_SETTINGS %}
{% endif %} {% endif %}
include:
- elasticsearch.enabled
escomponenttemplates: escomponenttemplates:
file.recurse: file.recurse:
- name: /opt/so/conf/elasticsearch/templates/component - name: /opt/so/conf/elasticsearch/templates/component
@@ -38,20 +35,6 @@ so_index_template_dir:
{%- endfor %} {%- endfor %}
{%- endif %} {%- endif %}
{% if GLOBALS.role != "so-heavynode" %}
# Clean up legacy and non-SO managed templates from the elasticsearch/templates/addon-index/ directory
addon_index_template_dir:
file.directory:
- name: /opt/so/conf/elasticsearch/templates/addon-index
- clean: True
{%- if ADDON_INDICES %}
- require:
{%- for index in ADDON_INDICES %}
- file: addon_index_template_{{index}}
{%- endfor %}
{%- endif %}
{% endif %}
# Auto-generate index templates for SO managed indices (directly defined in elasticsearch/defaults.yaml) # Auto-generate index templates for SO managed indices (directly defined in elasticsearch/defaults.yaml)
# These index templates are for the core SO datasets and are always required # These index templates are for the core SO datasets and are always required
{% for index, settings in ES_INDEX_SETTINGS.items() %} {% for index, settings in ES_INDEX_SETTINGS.items() %}
+3 -22
View File
@@ -61,25 +61,15 @@
{% if ALL_ADDON_SETTINGS_ORIG.keys() | length > 0 %} {% if ALL_ADDON_SETTINGS_ORIG.keys() | length > 0 %}
{% for index in ALL_ADDON_SETTINGS_ORIG.keys() %} {% for index in ALL_ADDON_SETTINGS_ORIG.keys() %}
{% do ALL_ADDON_SETTINGS_GLOBAL_OVERRIDES.update({index: salt['defaults.merge'](ALL_ADDON_SETTINGS_ORIG[index], PILLAR_GLOBAL_OVERRIDES, in_place=False)}) %} {% do ALL_ADDON_SETTINGS_GLOBAL_OVERRIDES.update({index: salt['defaults.merge'](ALL_ADDON_SETTINGS_ORIG[index], PILLAR_GLOBAL_OVERRIDES, in_place=False)}) %}
{# Explicitly excluding addon indices from ES_INDEX_SETTINGS_ORIG
When manager.soc_managed_annotations runs, new entries are added to the salt/elasticsearch/defaults.yaml file to support 'revert to default' functionality.
Subsequent map renders will then incorrectly include 'integration X' in 'ES_INDEX_SETTINGS_ORIG' due to being in the defaults.yaml file. #}
{% if index in ES_INDEX_SETTINGS_ORIG.keys() %}
{% do ES_INDEX_SETTINGS_ORIG.pop(index) %}
{% endif %}
{% endfor %} {% endfor %}
{% endif %} {% endif %}
{% set ES_INDEX_SETTINGS = {} %} {% set ES_INDEX_SETTINGS = {} %}
{% macro create_final_index_template(DEFINED_SETTINGS, GLOBAL_OVERRIDES, FINAL_INDEX_SETTINGS, EXCLUDE_INDICES=[]) %} {% macro create_final_index_template(DEFINED_SETTINGS, GLOBAL_OVERRIDES, FINAL_INDEX_SETTINGS) %}
{% do GLOBAL_OVERRIDES.update(salt['defaults.merge'](GLOBAL_OVERRIDES, ES_INDEX_PILLAR, in_place=False)) %} {% do GLOBAL_OVERRIDES.update(salt['defaults.merge'](GLOBAL_OVERRIDES, ES_INDEX_PILLAR, in_place=False)) %}
{% for index, settings in GLOBAL_OVERRIDES.items() %} {% for index, settings in GLOBAL_OVERRIDES.items() %}
{% if index in EXCLUDE_INDICES %}
{% continue %}
{% endif %}
{# prevent this action from being performed on custom defined indices. #} {# prevent this action from being performed on custom defined indices. #}
{# the custom defined index is not present in either of the dictionaries and fails to reder. #} {# the custom defined index is not present in either of the dictionaries and fails to reder. #}
{% if index in DEFINED_SETTINGS and index in GLOBAL_OVERRIDES %} {% if index in DEFINED_SETTINGS and index in GLOBAL_OVERRIDES %}
@@ -160,19 +150,10 @@
{% endfor %} {% endfor %}
{% endmacro %} {% endmacro %}
{# Exclude addon integrations from final ES_INDEX_SETTINGS #} {{ create_final_index_template(ES_INDEX_SETTINGS_ORIG, ES_INDEX_SETTINGS_GLOBAL_OVERRIDES, ES_INDEX_SETTINGS) }}
{{ create_final_index_template(ES_INDEX_SETTINGS_ORIG, ES_INDEX_SETTINGS_GLOBAL_OVERRIDES, ES_INDEX_SETTINGS, ALL_ADDON_SETTINGS_ORIG.keys() | list ) }} {{ create_final_index_template(ALL_ADDON_SETTINGS_ORIG, ALL_ADDON_SETTINGS_GLOBAL_OVERRIDES, ALL_ADDON_SETTINGS) }}
{# Exclude SO managed indices, otherwise ALL_ADDON_SETTINGS will include pillar values
of core integrations without merging defaults, resulting in an overlapping, but bad index template being generated. #}
{{ create_final_index_template(ALL_ADDON_SETTINGS_ORIG, ALL_ADDON_SETTINGS_GLOBAL_OVERRIDES, ALL_ADDON_SETTINGS, ES_INDEX_SETTINGS_ORIG.keys() | list ) }}
{% set SO_MANAGED_INDICES = [] %} {% set SO_MANAGED_INDICES = [] %}
{% for index, settings in ES_INDEX_SETTINGS.items() %} {% for index, settings in ES_INDEX_SETTINGS.items() %}
{% do SO_MANAGED_INDICES.append(index) %} {% do SO_MANAGED_INDICES.append(index) %}
{% endfor %} {% endfor %}
{% set ADDON_INDICES = [] %}
{% for index, settings in ALL_ADDON_SETTINGS.items() %}
{% do ADDON_INDICES.append(index) %}
{% endfor %}
-14
View File
@@ -6,7 +6,6 @@
{% from 'allowed_states.map.jinja' import allowed_states %} {% from 'allowed_states.map.jinja' import allowed_states %}
{% if sls.split('.')[0] in allowed_states %} {% if sls.split('.')[0] in allowed_states %}
{% from 'docker/docker.map.jinja' import DOCKERMERGED %} {% from 'docker/docker.map.jinja' import DOCKERMERGED %}
{% from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
{% from 'vars/globals.map.jinja' import GLOBALS %} {% from 'vars/globals.map.jinja' import GLOBALS %}
include: include:
@@ -61,19 +60,6 @@ so-kibana:
- watch: - watch:
- file: kibanaconfig - file: kibanaconfig
wait_for_so-kibana:
http.wait_for_successful_query:
- name: "http://localhost:5601/api/status"
- username: 'so_elastic'
- password: '{{ ELASTICSEARCHMERGED.auth.users.so_elastic_user.pass }}'
- ssl: True
- verify_ssl: False
- status: 200
- wait_for: 300
- request_interval: 15
- require:
- docker_container: so-kibana
delete_so-kibana_so-status.disabled: delete_so-kibana_so-status.disabled:
file.uncomment: file.uncomment:
- name: /opt/so/conf/so-status/so-status.conf - name: /opt/so/conf/so-status/so-status.conf
+1 -1
View File
@@ -103,7 +103,7 @@ kratos:
config: config:
session: session:
lifespan: lifespan:
description: Defines the length of a login session before it will timeout, and require a new login. description: Defines the length of a login session.
global: True global: True
helpLink: kratos helpLink: kratos
whoami: whoami:
+3 -5
View File
@@ -31,13 +31,11 @@ sync_es_users:
- http: wait_for_kratos - http: wait_for_kratos
- file: so-user.lock # require so-user.lock file to be missing - file: so-user.lock # require so-user.lock file to be missing
# we dont want this added too early in setup, so the onlyif gates on the # we dont want this added too early in setup, so we add the onlyif to verify 'startup_states: highstate'
# /opt/so/state/setup-complete marker. The marker is written by # is in the minion config. That line is added before the final highstate during setup
# mark_setup_complete in setup/so-functions just before the final setup
# highstate (and by an upgrade-path state for systems set up under the old gate).
so-user_sync: so-user_sync:
cron.present: cron.present:
- user: root - user: root
- name: 'PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin /usr/sbin/so-user sync &>> /opt/so/log/soc/sync.log' - name: 'PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin /usr/sbin/so-user sync &>> /opt/so/log/soc/sync.log'
- identifier: so-user_sync - identifier: so-user_sync
- onlyif: "test -e /opt/so/state/setup-complete" - onlyif: "grep -x 'startup_states: highstate' /etc/salt/minion"
-117
View File
@@ -1,117 +0,0 @@
#!/bin/bash
#
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
# Runs once per boot on managers (via so-boot-mine-update.service), before
# so-boot-highstate.service. Waits for the responsive minion set to settle, pushes
# mine.update, waits until every up minion has actually reported to the mine, then
# warms the master's per-minion pillar cache so the mine-backed node pillars (node
# IPs, ES/Redis/Logstash/hypervisor discovery -- some glob- and some pillar/grain-
# targeted) are complete before the boot highstate renders them. Otherwise a node
# that is up but not yet fully reported gets dropped from those pillars and torn
# out of the configs they build (e.g. so-elasticsearch ExtraHosts -> container recreate).
MAX_WAIT=${MINE_UPDATE_MAX_WAIT:-180} # hard backstop only
INTERVAL=10
STABLE_CHECKS=3 # up-count must hold steady this many polls
elapsed=0
prev=-1
stable=0
up=0
# Wait for the *reachable* minion set to settle rather than for every accepted
# key to report up: an operator may accept a minion's key and then intentionally
# power off that host, so requiring up >= accepted would never be satisfied and
# we'd always burn the full MAX_WAIT. Once the responsive count stops growing we
# stop waiting and run mine.update against whoever is up.
while [ "$elapsed" -lt "$MAX_WAIT" ]; do
up=$(/usr/bin/salt-run manage.up --out=json 2>/dev/null \
| python3 -c 'import sys,json; print(len(json.load(sys.stdin)))' 2>/dev/null)
up=${up:-0}
if [ "$up" -gt 0 ] && [ "$up" -eq "$prev" ]; then
stable=$((stable + 1))
[ "$stable" -ge "$STABLE_CHECKS" ] && break
else
stable=0
fi
prev=$up
sleep "$INTERVAL"
elapsed=$((elapsed + INTERVAL))
done
echo "so-boot-mine-update: ${up} minions up (settled after ${elapsed}s); running mine.update"
/usr/bin/salt '*' mine.update --out=txt
# A node that is up but has not yet re-reported network.ip_addrs to the mine is
# silently dropped from mine-backed pillars (elasticsearch:nodes, node_data, ...)
# when highstate recompiles them -- which e.g. removes it from so-elasticsearch
# ExtraHosts and forces a container recreate. After the broad mine.update above,
# wait until every up minion actually has network.ip_addrs in the mine, re-pushing
# mine.update to stragglers, before releasing the boot highstate. Bounded by the
# same MAX_WAIT backstop so a slow/down node never blocks boot indefinitely.
missing=""
while [ "$elapsed" -lt "$MAX_WAIT" ]; do
up_json=$(/usr/bin/salt-run manage.up --out=json 2>/dev/null)
mine_json=$(/usr/bin/salt-run mine.get '*' network.ip_addrs tgt_type=glob --out=json 2>/dev/null)
missing=$(printf '%s' "$up_json" | python3 -c '
import sys, json
up = set(json.load(sys.stdin) or [])
mine = {k for k, v in (json.loads(sys.argv[1]) or {}).items() if v}
print("\n".join(sorted(up - mine)))
' "$mine_json" 2>/dev/null)
if [ -z "$missing" ]; then
echo "so-boot-mine-update: mine complete for all up minions after ${elapsed}s"
break
fi
echo "so-boot-mine-update: mine missing up minion(s): $(echo $missing); re-running mine.update"
for m in $missing; do /usr/bin/salt "$m" mine.update --out=txt; done
sleep "$INTERVAL"
elapsed=$((elapsed + INTERVAL))
done
[ -n "$missing" ] && echo "so-boot-mine-update: WARNING ${MAX_WAIT}s backstop hit; up minion(s) still absent from mine: $(echo $missing); highstate may drop them from configs"
# The pillar/compound-targeted node pillars (elasticsearch:nodes, redis:nodes,
# logstash:nodes, hypervisor:nodes) resolve their target against the master's
# per-minion data cache (grains+pillar in .../minions/<id>/data.p), populated only
# when a minion's pillar is (re)compiled -- separately from the mine. A freshly
# booted node can be in the mine (glob/node_data sees it) yet absent from that
# cache, so it is dropped from those pillars and from the configs they build (e.g.
# so-elasticsearch ExtraHosts). Force a synchronous pillar refresh so the master
# caches every up node's pillar; refresh_pillar wait=True returns only once the
# pillar is recompiled (and thus cached for matching). Retry stragglers <= MAX_WAIT.
echo "so-boot-mine-update: warming master pillar cache for pillar/grain-targeted node pillars"
/usr/bin/salt '*' saltutil.refresh_pillar wait=True --out=txt
missing=""
while [ "$elapsed" -lt "$MAX_WAIT" ]; do
up_json=$(/usr/bin/salt-run manage.up --out=json 2>/dev/null)
cached_json=$(/usr/bin/salt-run cache.pillar tgt='*' --out=json 2>/dev/null)
missing=$(printf '%s' "$up_json" | python3 -c '
import sys, json
up = set(json.load(sys.stdin) or [])
cached = {k for k, v in (json.loads(sys.argv[1]) or {}).items() if v}
print("\n".join(sorted(up - cached)))
' "$cached_json" 2>/dev/null)
if [ -z "$missing" ]; then
echo "so-boot-mine-update: pillar cache warm for all up minions after ${elapsed}s"
break
fi
echo "so-boot-mine-update: pillar not yet cached for: $(echo $missing); refreshing"
for m in $missing; do /usr/bin/salt "$m" saltutil.refresh_pillar wait=True --out=txt; done
sleep "$INTERVAL"
elapsed=$((elapsed + INTERVAL))
done
[ -n "$missing" ] && echo "so-boot-mine-update: WARNING ${MAX_WAIT}s backstop hit; pillar not cached for: $(echo $missing); pillar-targeted pillars may drop them"
# Log what the mine-backed pillars render so the boot-time state is inspectable.
/usr/bin/salt-call saltutil.refresh_pillar >/dev/null 2>&1
sleep 2
for key in node_data elasticsearch:nodes; do
rendered=$(/usr/bin/salt-call --out=json pillar.get "$key" 2>/dev/null \
| python3 -c 'import sys,json; print(json.dumps(json.load(sys.stdin).get("local"), indent=2, sort_keys=True))' 2>/dev/null)
echo "so-boot-mine-update: ${key} rendered as:"
echo "${rendered:-null}"
done
exit 0
+19 -51
View File
@@ -188,6 +188,13 @@ airgap_update_dockers() {
fi fi
} }
backup_old_states_pillars() {
tar czf /nsm/backup/$(echo $INSTALLEDVERSION)_$(date +%Y%m%d-%H%M%S)_soup_default_states_pillars.tar.gz /opt/so/saltstack/default/
tar czf /nsm/backup/$(echo $INSTALLEDVERSION)_$(date +%Y%m%d-%H%M%S)_soup_local_states_pillars.tar.gz /opt/so/saltstack/local/
}
update_registry() { update_registry() {
docker stop so-dockerregistry docker stop so-dockerregistry
docker rm so-dockerregistry docker rm so-dockerregistry
@@ -365,7 +372,6 @@ preupgrade_changes() {
[[ "$INSTALLEDVERSION" =~ ^2\.4\.21[0-9]+$ ]] && up_to_3.0.0 [[ "$INSTALLEDVERSION" =~ ^2\.4\.21[0-9]+$ ]] && up_to_3.0.0
[[ "$INSTALLEDVERSION" == "3.0.0" ]] && up_to_3.1.0 [[ "$INSTALLEDVERSION" == "3.0.0" ]] && up_to_3.1.0
[[ "$INSTALLEDVERSION" == "3.1.0" ]] && up_to_3.2.0
true true
} }
@@ -375,7 +381,6 @@ postupgrade_changes() {
[[ "$POSTVERSION" =~ ^2\.4\.21[0-9]+$ ]] && post_to_3.0.0 [[ "$POSTVERSION" =~ ^2\.4\.21[0-9]+$ ]] && post_to_3.0.0
[[ "$POSTVERSION" == "3.0.0" ]] && post_to_3.1.0 [[ "$POSTVERSION" == "3.0.0" ]] && post_to_3.1.0
[[ "$POSTVERSION" == "3.1.0" ]] && post_to_3.2.0
true true
} }
@@ -737,48 +742,6 @@ post_to_3.1.0() {
### 3.1.0 End ### ### 3.1.0 End ###
### 3.2.0 Scripts ###
bootstrap_so_soc_database() {
# init-db.sh is mounted into so-postgres at /docker-entrypoint-initdb.d/init-db.sh
# and runs automatically only on a fresh data directory. Hosts upgrading from
# 3.1.0 already have /nsm/postgres populated, so the so_soc bootstrap block
# added in 3.2 never fires. Re-run the script explicitly; it's idempotent.
echo "Bootstrapping so_soc database via init-db.sh."
# The postgres image has no USER directive, so `docker exec` defaults to
# root, and the container env intentionally omits POSTGRES_USER (the upstream
# entrypoint defaults it transiently during first-init only). Recreate both
# so psql inside init-db.sh resolves the connect user correctly.
local exec_cmd="docker exec -u postgres -e POSTGRES_USER=postgres so-postgres bash /docker-entrypoint-initdb.d/init-db.sh"
if ! /usr/sbin/so-postgres-wait; then
FINAL_MESSAGE_QUEUE+=("WARNING: so-postgres was not ready during the 3.2.0 upgrade; the so_soc database may not have been bootstrapped. Re-run manually: $exec_cmd")
return 0
fi
if ! $exec_cmd; then
FINAL_MESSAGE_QUEUE+=("WARNING: init-db.sh failed inside so-postgres during the 3.2.0 upgrade; the so_soc database may not have been bootstrapped. Re-run manually: $exec_cmd")
return 0
fi
echo "so_soc bootstrap complete."
}
up_to_3.2.0() {
fix_logstash_0013_lumberjack_pipeline_name
INSTALLEDVERSION=3.2.0
}
post_to_3.2.0() {
bootstrap_so_soc_database
# Including agent regen script here since it was missed in post_to_3.1.0
echo "Regenerating Elastic Agent Installers"
/sbin/so-elastic-agent-gen-installers
POSTVERSION=3.2.0
}
### 3.2.0 End ###
repo_sync() { repo_sync() {
echo "Sync the local repo." echo "Sync the local repo."
@@ -1565,7 +1528,13 @@ EOF
# Keeping this block in case we need to do a hotfix that requires salt update # Keeping this block in case we need to do a hotfix that requires salt update
apply_hotfix() { apply_hotfix() {
if [[ "$INSTALLEDVERSION" == "3.1.0" ]] ; then
# Do not remove this fix_logstash_0013_lumberjack_pipeline_name in future hotfixes without first validating older
# installs referencing "so/0013_input_lumberjack_fleet.conf" via pillar are upgradable
fix_logstash_0013_lumberjack_pipeline_name
else
echo "No actions required. ($INSTALLEDVERSION/$HOTFIXVERSION)" echo "No actions required. ($INSTALLEDVERSION/$HOTFIXVERSION)"
fi
} }
failed_soup_restore_items() { failed_soup_restore_items() {
@@ -1637,13 +1606,13 @@ main() {
echo "Verifying we have the latest soup script." echo "Verifying we have the latest soup script."
verify_latest_update_script verify_latest_update_script
echo "Verifying Elasticsearch version compatibility across the grid before upgrading."
verify_es_version_compatibility
echo "Let's see if we need to update Security Onion." echo "Let's see if we need to update Security Onion."
upgrade_check upgrade_check
upgrade_space upgrade_space
echo "Verifying Elasticsearch version compatibility across the grid before upgrading."
verify_es_version_compatibility
echo "Checking for Salt Master and Minion updates." echo "Checking for Salt Master and Minion updates."
upgrade_check_salt upgrade_check_salt
set -e set -e
@@ -1663,8 +1632,7 @@ main() {
echo "Applying $HOTFIXVERSION hotfix" echo "Applying $HOTFIXVERSION hotfix"
# since we don't run the backup.config_backup state on import we wont snapshot previous version states and pillars # since we don't run the backup.config_backup state on import we wont snapshot previous version states and pillars
if [[ ! "$MINION_ROLE" == "import" ]]; then if [[ ! "$MINION_ROLE" == "import" ]]; then
echo "Running so-config-backup script." backup_old_states_pillars
/sbin/so-config-backup
fi fi
copy_new_files copy_new_files
create_local_directories "/opt/so/saltstack/default" create_local_directories "/opt/so/saltstack/default"
@@ -1720,8 +1688,8 @@ main() {
# since we don't run the backup.config_backup state on import we wont snapshot previous version states and pillars # since we don't run the backup.config_backup state on import we wont snapshot previous version states and pillars
if [[ ! "$MINION_ROLE" == "import" ]]; then if [[ ! "$MINION_ROLE" == "import" ]]; then
echo "" echo ""
echo "Running so-config-backup script." echo "Creating snapshots of default and local Salt states and pillars and saving to /nsm/backup/"
/sbin/so-config-backup backup_old_states_pillars
fi fi
echo "" echo ""
-1
View File
@@ -17,7 +17,6 @@ psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-E
END IF; END IF;
END END
\$\$; \$\$;
GRANT ALL ON SCHEMA public TO "$SO_POSTGRES_USER";
GRANT ALL PRIVILEGES ON DATABASE "$POSTGRES_DB" TO "$SO_POSTGRES_USER"; GRANT ALL PRIVILEGES ON DATABASE "$POSTGRES_DB" TO "$SO_POSTGRES_USER";
-- Lock the SOC database down at the connect layer; PUBLIC gets CONNECT -- Lock the SOC database down at the connect layer; PUBLIC gets CONNECT
-- by default, which would let per-minion telegraf roles open sessions -- by default, which would let per-minion telegraf roles open sessions
+16 -2
View File
@@ -18,12 +18,26 @@ include:
{% set TG_OUT = TELEGRAFMERGED.output | upper %} {% set TG_OUT = TELEGRAFMERGED.output | upper %}
{% if TG_OUT in ['POSTGRES', 'BOTH'] %} {% if TG_OUT in ['POSTGRES', 'BOTH'] %}
# docker_container.running returns as soon as the container starts, but on
# first-init docker-entrypoint.sh starts a temporary postgres with
# `listen_addresses=''` to run /docker-entrypoint-initdb.d scripts, then
# shuts it down before exec'ing the real CMD. A default pg_isready check
# (Unix socket) passes during that ephemeral phase and races the shutdown
# with "the database system is shutting down". Checking TCP readiness on
# 127.0.0.1 only succeeds after the final postgres binds the port.
postgres_wait_ready: postgres_wait_ready:
cmd.run: cmd.run:
- name: /usr/sbin/so-postgres-wait - name: |
for i in $(seq 1 60); do
if docker exec so-postgres pg_isready -h 127.0.0.1 -U postgres -q 2>/dev/null; then
exit 0
fi
sleep 2
done
echo "so-postgres did not accept TCP connections within 120s" >&2
exit 1
- require: - require:
- docker_container: so-postgres - docker_container: so-postgres
- file: postgres_sbin
# Ensure the shared Telegraf database exists. init-db.sh only runs on a # Ensure the shared Telegraf database exists. init-db.sh only runs on a
# fresh data dir, so hosts upgraded onto an existing /nsm/postgres volume # fresh data dir, so hosts upgraded onto an existing /nsm/postgres volume
-32
View File
@@ -1,32 +0,0 @@
#!/bin/bash
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
# Wait for the so-postgres container to accept TCP connections.
#
# docker_container.running returns as soon as the container starts, but on
# first-init docker-entrypoint.sh starts a temporary postgres with
# `listen_addresses=''` to run /docker-entrypoint-initdb.d scripts, then
# shuts it down before exec'ing the real CMD. A default pg_isready check
# (Unix socket) passes during that ephemeral phase and races the shutdown
# with "the database system is shutting down". Checking TCP readiness on
# 127.0.0.1 only succeeds after the final postgres binds the port.
#
# Usage: so-postgres-wait [iterations] [sleep_seconds]
# Default: 60 iterations, 2s sleep (~120s total).
ITERATIONS=${1:-60}
SLEEP_SECONDS=${2:-2}
for i in $(seq 1 "$ITERATIONS"); do
if docker exec so-postgres pg_isready -h 127.0.0.1 -U postgres -q 2>/dev/null; then
exit 0
fi
sleep "$SLEEP_SECONDS"
done
echo "so-postgres did not accept TCP connections within $((ITERATIONS * SLEEP_SECONDS))s" >&2
exit 1
+2 -1
View File
@@ -1,5 +1,6 @@
{% from 'vars/globals.map.jinja' import GLOBALS %} {% from 'vars/globals.map.jinja' import GLOBALS %}
{% if GLOBALS.os == 'OEL' %} {# OL10 test path uses public repos; skip the SO repo state (which removes public repos and points at /nsm/repo) #}
{% if GLOBALS.os == 'OEL' and GLOBALS.os_version|int == 9 %}
include: include:
- repo.client.oracle - repo.client.oracle
{% endif %} {% endif %}
-1
View File
@@ -14,7 +14,6 @@
include: include:
- salt.minion - salt.minion
- salt.master.boot_mine_update
{% if 'vrt' in salt['pillar.get']('features', []) %} {% if 'vrt' in salt['pillar.get']('features', []) %}
- salt.cloud - salt.cloud
- salt.cloud.reactor_config_hypervisor - salt.cloud.reactor_config_hypervisor
-29
View File
@@ -1,29 +0,0 @@
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
# Manages /etc/systemd/system/so-boot-mine-update.service, a manager-only
# Type=oneshot unit that pushes `salt '*' mine.update` once per boot, ordered
# before so-boot-highstate.service so mine-backed pillars (node IPs, ES/Redis/
# Logstash discovery) are fresh before the boot highstate renders them.
include:
- systemd.reload
so_boot_mine_update_unit_file:
file.managed:
- name: /etc/systemd/system/so-boot-mine-update.service
- source: salt://salt/service/so-boot-mine-update.service
- onchanges_in:
- module: systemd_reload
# Only enable once setup is complete. Until then the gate file is missing and
# the unit's own ConditionPathExists would no-op it anyway.
so_boot_mine_update_service:
service.enabled:
- name: so-boot-mine-update.service
- onlyif: test -e /opt/so/state/setup-complete
- require:
- file: so_boot_mine_update_unit_file
- module: systemd_reload
-31
View File
@@ -1,31 +0,0 @@
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
# Manages /etc/systemd/system/so-boot-highstate.service, a Type=oneshot
# RemainAfterExit=yes unit that runs `salt-call state.highstate` exactly once
# per system boot. Replaces the legacy `startup_states: highstate` minion
# config, which fired on every salt-minion service restart (causing a redundant
# highstate whenever a highstate itself restarted salt-minion).
include:
- systemd.reload
so_boot_highstate_unit_file:
file.managed:
- name: /etc/systemd/system/so-boot-highstate.service
- source: salt://salt/service/so-boot-highstate.service
- onchanges_in:
- module: systemd_reload
# Only enable once setup is complete. Until then the gate file is missing and
# the unit's own ConditionPathExists would no-op it anyway -- this just keeps
# `systemctl is-enabled` honest for the sync_es_users gate.
so_boot_highstate_service:
service.enabled:
- name: so-boot-highstate.service
- onlyif: test -e /opt/so/state/setup-complete
- require:
- file: so_boot_highstate_unit_file
- module: systemd_reload
+4 -27
View File
@@ -17,7 +17,6 @@ include:
- repo.client - repo.client
- salt.mine_functions - salt.mine_functions
- salt.minion.service_file - salt.minion.service_file
- salt.minion.boot_highstate
{% if GLOBALS.is_manager %} {% if GLOBALS.is_manager %}
- ca.signing_policy - ca.signing_policy
{% endif %} {% endif %}
@@ -81,33 +80,11 @@ set_log_levels:
- "log_level: info" - "log_level: info"
- "log_level_logfile: info" - "log_level_logfile: info"
# startup_states: highstate caused a full highstate to run on every enable_startup_states:
# salt-minion service start, including the restart triggered when a highstate file.uncomment:
# itself modified the minion config (beacons, mine, unit file). Replaced by
# so-boot-highstate.service (managed in salt.minion.boot_highstate), which
# runs once per system boot only. Strip the line from /etc/salt/minion on
# upgrade; both the commented and uncommented forms historically existed.
remove_startup_states:
file.line:
- name: /etc/salt/minion - name: /etc/salt/minion
- match: 'startup_states: highstate' - regex: '^startup_states: highstate$'
- mode: delete - unless: pgrep so-setup
# Upgrade-path bridge: systems that already passed setup under the old gate
# (`grep -x 'startup_states: highstate' /etc/salt/minion`) get a /opt/so/state/setup-complete
# marker so so-boot-highstate.service can be enabled and the so-user_sync cron
# in sync_es_users.sls keeps installing. Setup-in-progress systems instead get
# the marker from `mark_setup_complete` in setup/so-functions at the right
# moment. `replace: false` means we never overwrite a marker once written.
mark_setup_complete_for_upgrades:
file.managed:
- name: /opt/so/state/setup-complete
- replace: false
- makedirs: True
- onlyif: "grep -qx 'startup_states: highstate' /etc/salt/minion"
- require_in:
- file: remove_startup_states
- service: so_boot_highstate_service
{% endif %} {% endif %}
@@ -1,14 +0,0 @@
[Unit]
Description=Security Onion boot-time highstate (runs once per boot)
After=salt-minion.service network-online.target docker.service
Wants=network-online.target docker.service
Requires=salt-minion.service
ConditionPathExists=/opt/so/state/setup-complete
[Service]
Type=oneshot
RemainAfterExit=yes
ExecStart=/usr/bin/salt-call state.highstate -l info queue=True
[Install]
WantedBy=multi-user.target
@@ -1,15 +0,0 @@
[Unit]
Description=Security Onion boot-time grid mine.update (managers, runs once per boot before highstate)
After=salt-master.service salt-minion.service network-online.target
Wants=network-online.target
Requires=salt-master.service salt-minion.service
Before=so-boot-highstate.service
ConditionPathExists=/opt/so/state/setup-complete
[Service]
Type=oneshot
RemainAfterExit=yes
ExecStart=/usr/sbin/so-boot-mine-update
[Install]
WantedBy=multi-user.target
+5
View File
@@ -8,6 +8,11 @@ set_role_grain:
- name: role - name: role
- value: so-{{ grains.id.split("_") | last }} - value: so-{{ grains.id.split("_") | last }}
set_highstate:
file.append:
- name: /etc/salt/minion
- text: 'startup_states: highstate'
enable_salt_minion: enable_salt_minion:
service.enabled: service.enabled:
- name: salt-minion - name: salt-minion
-10
View File
@@ -1519,16 +1519,6 @@ soc:
serviceAccountJSON: "" serviceAccountJSON: ""
serviceAccountLocation: "" serviceAccountLocation: ""
healthTimeoutSeconds: 5 healthTimeoutSeconds: 5
onionconfig:
saltstackDir: /opt/so/saltstack
bypassEnabled: false
postgres:
host: ""
port: 5432
sslMode: "allow"
database: securityonion
user: ""
password: ""
salt: salt:
queueDir: /opt/sensoroni/queue queueDir: /opt/sensoroni/queue
timeoutMs: 45000 timeoutMs: 45000
-8
View File
@@ -16,14 +16,6 @@
{% do SOCMERGED.config.server.update({'additionalCA': MANAGERMERGED.additionalCA}) %} {% do SOCMERGED.config.server.update({'additionalCA': MANAGERMERGED.additionalCA}) %}
{% do SOCMERGED.config.server.update({'insecureSkipVerify': MANAGERMERGED.insecureSkipVerify}) %} {% do SOCMERGED.config.server.update({'insecureSkipVerify': MANAGERMERGED.insecureSkipVerify}) %}
{% if not SOCMERGED.config.server.modules.postgres.host %}
{% do SOCMERGED.config.server.modules.postgres.update({'host': GLOBALS.manager}) %}
{% endif %}
{% if not SOCMERGED.config.server.modules.postgres.password %}
{% do SOCMERGED.config.server.modules.postgres.update({'password': salt['pillar.get']('postgres:auth:users:so_postgres_user:pass', '')}) %}
{% do SOCMERGED.config.server.modules.postgres.update({'user': salt['pillar.get']('postgres:auth:users:so_postgres_user:user', 'so_postgres')}) %}
{% endif %}
{# if SOCMERGED.config.server.modules.cases == httpcase details come from the soc pillar #} {# if SOCMERGED.config.server.modules.cases == httpcase details come from the soc pillar #}
{% if SOCMERGED.config.server.modules.cases != 'soc' %} {% if SOCMERGED.config.server.modules.cases != 'soc' %}
{% do SOCMERGED.config.server.modules.elastic.update({'casesEnabled': false}) %} {% do SOCMERGED.config.server.modules.elastic.update({'casesEnabled': false}) %}
-37
View File
@@ -453,42 +453,6 @@ soc:
description: Duration (in milliseconds) that must elapse after a grid node fails to check-in before the node will be marked offline (fault). description: Duration (in milliseconds) that must elapse after a grid node fails to check-in before the node will be marked offline (fault).
global: True global: True
advanced: True advanced: True
onionconfig:
saltstackDir:
description: Root directory containing the SaltStack tree that SOC reads and writes configuration from. Should not be changed under normal circumstances.
global: True
advanced: True
bypassEnabled:
description: When enabled, errors encountered while reading the SaltStack pillar tree (missing files, unreadable directories, etc.) are logged but do not prevent SOC from starting or serving settings. Intended for advanced troubleshooting and recovery scenarios when the pillar tree is partially unreadable.
global: True
advanced: True
forcedType: bool
postgres:
host:
description: Hostname or IP address of the PostgreSQL server used by SOC. Defaults to the manager hostname.
global: True
advanced: True
port:
description: Port of the PostgreSQL server used by SOC.
global: True
advanced: True
sslMode:
description: "Use encrypted connections to the PostgreSQL server. Must be one of the following values: disable, allow, prefer, require, verify-ca, verify-full. Defaults to allow."
global: True
advanced: True
database:
description: Database used by SOC to authenticate to the PostgreSQL server.
global: True
advanced: True
user:
description: Username used by SOC to authenticate to the PostgreSQL server.
global: True
advanced: True
password:
description: Password used by SOC to authenticate to the PostgreSQL server.
global: True
sensitive: True
advanced: True
salt: salt:
longRelayTimeoutMs: longRelayTimeoutMs:
description: Duration (in milliseconds) to wait for a response from the Salt API when executing tasks known for being long running before giving up and showing an error on the SOC UI. description: Duration (in milliseconds) to wait for a response from the Salt API when executing tasks known for being long running before giving up and showing an error on the SOC UI.
@@ -854,7 +818,6 @@ soc:
description: List of available external tools visible in the SOC UI. Each tool is defined in JSON object notation, and must include the "name" key and "link" key, where the link is the tool's URL. description: List of available external tools visible in the SOC UI. Each tool is defined in JSON object notation, and must include the "name" key and "link" key, where the link is the tool's URL.
global: True global: True
advanced: True advanced: True
multiline: True
forcedType: "[]{}" forcedType: "[]{}"
exportNodeId: exportNodeId:
description: The node ID on which export jobs will be executed. description: The node ID on which export jobs will be executed.
+1
View File
@@ -31,6 +31,7 @@
'so_model': INIT.GRAINS.get('sosmodel',''), 'so_model': INIT.GRAINS.get('sosmodel',''),
'sensoroni_key': INIT.PILLAR.sensoroni.config.sensoronikey, 'sensoroni_key': INIT.PILLAR.sensoroni.config.sensoronikey,
'os': INIT.GRAINS.os, 'os': INIT.GRAINS.os,
'os_version': INIT.GRAINS.osmajorrelease,
'os_family': INIT.GRAINS.os_family, 'os_family': INIT.GRAINS.os_family,
'application_urls': {}, 'application_urls': {},
'manager_roles': [ 'manager_roles': [
+27 -16
View File
@@ -539,19 +539,16 @@ configure_minion() {
" x509_v2: true"\ " x509_v2: true"\
"log_level: info"\ "log_level: info"\
"log_level_logfile: info"\ "log_level_logfile: info"\
"log_file: /opt/so/log/salt/minion" >> "$minion_config" "log_file: /opt/so/log/salt/minion"\
"#startup_states: highstate" >> "$minion_config"
} }
mark_setup_complete() { checkin_at_boot() {
# Writes the setup-complete marker. Salt's so-boot-highstate.service local minion_config=/etc/salt/minion
# (boot-time oneshot) and the so-user_sync cron gate in
# salt/manager/sync_es_users.sls both key off this file.
local marker=/opt/so/state/setup-complete
info "Marking setup as complete" info "Enabling checkin at boot"
mkdir -p "$(dirname "$marker")" sed -i 's/#startup_states: highstate/startup_states: highstate/' "$minion_config"
touch "$marker"
} }
check_requirements() { check_requirements() {
@@ -906,14 +903,14 @@ detect_cloud() {
detect_os() { detect_os() {
title "Detecting Base OS" title "Detecting Base OS"
if [ -f /etc/redhat-release ] && grep -q "Red Hat Enterprise Linux release 9" /etc/redhat-release && [ -f /etc/oracle-release ]; then if [ -f /etc/oracle-release ] && grep -qE "release (9|10)\b" /etc/oracle-release; then
OS=oracle OS=oracle
OSVER=9 OSVER=$(grep -oE "release [0-9]+" /etc/oracle-release | grep -oE "[0-9]+")
is_oracle=true is_oracle=true
is_rpm=true is_rpm=true
is_supported=true is_supported=true
else else
info "This OS is not supported. Security Onion requires Oracle Linux 9." info "This OS is not supported. Security Onion requires Oracle Linux 9 or 10."
fail_setup fail_setup
fi fi
@@ -980,8 +977,6 @@ docker_seed_registry() {
docker_seed_update_percent=25 docker_seed_update_percent=25
update_docker_containers 'netinstall' '' 'docker_seed_update' '/dev/stdout' 2>&1 | tee -a "$setup_log" update_docker_containers 'netinstall' '' 'docker_seed_update' '/dev/stdout' 2>&1 | tee -a "$setup_log"
# Use pipe exit status of 'update_docker_containers' for return code
return ${PIPESTATUS[0]}
fi fi
} }
@@ -1788,6 +1783,15 @@ ensure_pyyaml() {
# - securityonion/salt/salt/minion.defaults.yaml # - securityonion/salt/salt/minion.defaults.yaml
securityonion_repo() { securityonion_repo() {
if [[ "$OSVER" == "10" ]]; then
# TEST PATH: Oracle Linux 10 uses the public OL10 + EPEL + Docker CE repos.
# Keep the stock /etc/yum.repos.d/* in place, skip the SO mirror and local reposync.
gpg_rpm_import
logCmd "dnf -y install oracle-epel-release-el10"
logCmd "dnf -y config-manager --add-repo https://download.docker.com/linux/rhel/docker-ce.repo"
logCmd "dnf repolist"
return
fi
# Remove all the current repos # Remove all the current repos
logCmd "dnf -v clean all" logCmd "dnf -v clean all"
logCmd "mkdir -vp /root/oldrepos" logCmd "mkdir -vp /root/oldrepos"
@@ -1882,12 +1886,19 @@ saltify() {
info "Installing Salt $SALTVERSION" info "Installing Salt $SALTVERSION"
chmod u+x ../salt/salt/scripts/bootstrap-salt.sh chmod u+x ../salt/salt/scripts/bootstrap-salt.sh
# Normally Salt packages come from the SO mirror, so -r disables the bootstrap's own repo setup.
# On the OL10 test path there is no SO mirror, so let bootstrap configure the public Salt repo.
local saltrepoflag="-r"
if [[ "$OSVER" == "10" ]]; then
saltrepoflag=""
fi
if [[ $waitforstate ]]; then if [[ $waitforstate ]]; then
# install all for a manager # install all for a manager
retry 30 10 "bash ../salt/salt/scripts/bootstrap-salt.sh -r -M -X stable $SALTVERSION" || fail_setup retry 30 10 "bash ../salt/salt/scripts/bootstrap-salt.sh $saltrepoflag -M -X stable $SALTVERSION" || fail_setup
else else
# just a minion # just a minion
retry 30 10 "bash ../salt/salt/scripts/bootstrap-salt.sh -r -X stable $SALTVERSION" || fail_setup retry 30 10 "bash ../salt/salt/scripts/bootstrap-salt.sh $saltrepoflag -X stable $SALTVERSION" || fail_setup
fi fi
salt_install_module_deps salt_install_module_deps
+2 -7
View File
@@ -223,8 +223,6 @@ if [ -n "$test_profile" ]; then
WEBPASSWD1=0n10nus3r WEBPASSWD1=0n10nus3r
WEBPASSWD2=0n10nus3r WEBPASSWD2=0n10nus3r
NODE_DESCRIPTION="${HOSTNAME} - ${install_type} - ${MSRVIP_OFFSET}" NODE_DESCRIPTION="${HOSTNAME} - ${install_type} - ${MSRVIP_OFFSET}"
# opt out of telemetry for automated testing
telemetry=1
update_sudoers_for_testing update_sudoers_for_testing
fi fi
@@ -769,10 +767,7 @@ if ! [[ -f $install_opt_file ]]; then
title "Applying the registry state" title "Applying the registry state"
logCmd "salt-call state.apply -l info registry" logCmd "salt-call state.apply -l info registry"
title "Seeding the docker registry" title "Seeding the docker registry"
if ! docker_seed_registry; then docker_seed_registry
error "Failed to seed the docker registry"
fail_setup
fi
title "Applying the manager state" title "Applying the manager state"
logCmd "salt-call state.apply -l info manager" logCmd "salt-call state.apply -l info manager"
logCmd "salt-call state.apply influxdb -l info" logCmd "salt-call state.apply influxdb -l info"
@@ -797,7 +792,7 @@ if ! [[ -f $install_opt_file ]]; then
error "Failed to run so-elastic-fleet-setup" error "Failed to run so-elastic-fleet-setup"
fail_setup fail_setup
fi fi
mark_setup_complete checkin_at_boot
set_initial_firewall_access set_initial_firewall_access
initialize_elasticsearch_indices "so-case so-casehistory so-assistant-session so-assistant-chat" initialize_elasticsearch_indices "so-case so-casehistory so-assistant-session so-assistant-chat"
# run a final highstate before enabling scheduled highstates. # run a final highstate before enabling scheduled highstates.