CSEC_PUBLIC/securityonion
1
0
Fork 0
mirror of https://github.com/Security-Onion-Solutions/securityonion.git synced 2026-08-06 07:55:59 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #15804 from Security-Onion-Solutions/feature/postgres-integration

Browse Source 
postgres integration: sync feature/postgres to bravo for automated testing
This commit is contained in:
Mike Reeves
2026-04-21 11:14:56 -04:00
committed by GitHub
parent da69f0f1a4 05f6503d61
commit 922fc60466
24 changed files with 445 additions and 263 deletions
Download Patch File Download Diff File Expand all files Collapse all files
salt/backup/config_backup.sls
-19
View File
@@ -33,22 +33,3 @@ so_config_backup:
- month: '*'
- dayweek: '*'
postgres_backup_script:
file.managed:
- name: /usr/sbin/so-postgres-backup
- user: root
- group: root
- mode: 755
- source: salt://backup/tools/sbin/so-postgres-backup
# Add postgres database backup
so_postgres_backup:
cron.present:
- name: /usr/sbin/so-postgres-backup > /dev/null 2>&1
- identifier: so_postgres_backup
- user: root
- minute: '5'
- hour: '0'
- daymonth: '*'
- month: '*'
- dayweek: '*'
salt/docker/defaults.yaml
+1 -1
View File
@@ -238,7 +238,7 @@ docker:
extra_env: []
ulimits: []
'so-postgres':
final_octet: 89
final_octet: 47
port_bindings:
- 0.0.0.0:5432:5432
custom_bind_mounts: []
salt/elasticsearch/cluster.sls
+164
View File
@@ -0,0 +1,164 @@
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{% from 'allowed_states.map.jinja' import allowed_states %}
{% if sls.split('.')[0] in allowed_states %}
{% from 'vars/globals.map.jinja' import GLOBALS %}
{% from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
{% from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS, SO_MANAGED_INDICES %}
{% if GLOBALS.role != 'so-heavynode' %}
{% from 'elasticsearch/template.map.jinja' import ALL_ADDON_SETTINGS %}
{% endif %}
escomponenttemplates:
file.recurse:
- name: /opt/so/conf/elasticsearch/templates/component
- source: salt://elasticsearch/templates/component
- user: 930
- group: 939
- clean: True
- onchanges_in:
- file: so-elasticsearch-templates-reload
- show_changes: False
# Clean up legacy and non-SO managed templates from the elasticsearch/templates/index/ directory
so_index_template_dir:
file.directory:
- name: /opt/so/conf/elasticsearch/templates/index
- clean: True
{%- if SO_MANAGED_INDICES %}
- require:
{%- for index in SO_MANAGED_INDICES %}
- file: so_index_template_{{index}}
{%- endfor %}
{%- endif %}
# Auto-generate index templates for SO managed indices (directly defined in elasticsearch/defaults.yaml)
# These index templates are for the core SO datasets and are always required
{% for index, settings in ES_INDEX_SETTINGS.items() %}
{% if settings.index_template is defined %}
so_index_template_{{index}}:
file.managed:
- name: /opt/so/conf/elasticsearch/templates/index/{{ index }}-template.json
- source: salt://elasticsearch/base-template.json.jinja
- defaults:
TEMPLATE_CONFIG: {{ settings.index_template }}
- template: jinja
- onchanges_in:
- file: so-elasticsearch-templates-reload
{% endif %}
{% endfor %}
{% if GLOBALS.role != "so-heavynode" %}
# Auto-generate optional index templates for integration | input | content packages
# These index templates are not used by default (until user adds package to an agent policy).
# Pre-configured with standard defaults, and incorporated into SOC configuration for user customization.
{% for index,settings in ALL_ADDON_SETTINGS.items() %}
{% if settings.index_template is defined %}
addon_index_template_{{index}}:
file.managed:
- name: /opt/so/conf/elasticsearch/templates/addon-index/{{ index }}-template.json
- source: salt://elasticsearch/base-template.json.jinja
- defaults:
TEMPLATE_CONFIG: {{ settings.index_template }}
- template: jinja
- show_changes: False
- onchanges_in:
- file: addon-elasticsearch-templates-reload
{% endif %}
{% endfor %}
{% endif %}
{% if GLOBALS.role in GLOBALS.manager_roles %}
so-es-cluster-settings:
cmd.run:
- name: /usr/sbin/so-elasticsearch-cluster-settings
- cwd: /opt/so
- template: jinja
- require:
- docker_container: so-elasticsearch
- file: elasticsearch_sbin_jinja
- http: wait_for_so-elasticsearch
{% endif %}
# heavynodes will only load ILM policies for SO managed indices. (Indicies defined in elasticsearch/defaults.yaml)
so-elasticsearch-ilm-policy-load:
cmd.run:
- name: /usr/sbin/so-elasticsearch-ilm-policy-load
- cwd: /opt/so
- require:
- docker_container: so-elasticsearch
- file: so-elasticsearch-ilm-policy-load-script
- onchanges:
- file: so-elasticsearch-ilm-policy-load-script
so-elasticsearch-templates-reload:
file.absent:
- name: /opt/so/state/estemplates.txt
addon-elasticsearch-templates-reload:
file.absent:
- name: /opt/so/state/addon_estemplates.txt
# so-elasticsearch-templates-load will have its first successful run during the 'so-elastic-fleet-setup' script
so-elasticsearch-templates:
cmd.run:
{%- if GLOBALS.role == "so-heavynode" %}
- name: /usr/sbin/so-elasticsearch-templates-load --heavynode
{%- else %}
- name: /usr/sbin/so-elasticsearch-templates-load
{%- endif %}
- cwd: /opt/so
- template: jinja
- require:
- docker_container: so-elasticsearch
- file: elasticsearch_sbin_jinja
so-elasticsearch-pipelines:
cmd.run:
- name: /usr/sbin/so-elasticsearch-pipelines {{ GLOBALS.hostname }}
- require:
- docker_container: so-elasticsearch
- file: so-elasticsearch-pipelines-script
so-elasticsearch-roles-load:
cmd.run:
- name: /usr/sbin/so-elasticsearch-roles-load
- cwd: /opt/so
- template: jinja
- require:
- docker_container: so-elasticsearch
- file: elasticsearch_sbin_jinja
{% if grains.role in ['so-managersearch', 'so-manager', 'so-managerhype'] %}
{% set ap = "absent" %}
{% endif %}
{% if grains.role in ['so-eval', 'so-standalone', 'so-heavynode'] %}
{% if ELASTICSEARCHMERGED.index_clean %}
{% set ap = "present" %}
{% else %}
{% set ap = "absent" %}
{% endif %}
{% endif %}
{% if grains.role in ['so-eval', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-manager'] %}
so-elasticsearch-indices-delete:
cron.{{ap}}:
- name: /usr/sbin/so-elasticsearch-indices-delete > /opt/so/log/elasticsearch/cron-elasticsearch-indices-delete.log 2>&1
- identifier: so-elasticsearch-indices-delete
- user: root
- minute: '*/5'
- hour: '*'
- daymonth: '*'
- month: '*'
- dayweek: '*'
{% endif %}
{% else %}
{{sls}}_state_not_allowed:
test.fail_without_changes:
- name: {{sls}}_state_not_allowed
{% endif %}
salt/elasticsearch/enabled.sls
+14 -146
View File
@@ -10,10 +10,6 @@
{% from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_NODES %}
{% from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_SEED_HOSTS %}
{% from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
{% from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS, SO_MANAGED_INDICES %}
{% if GLOBALS.role != 'so-heavynode' %}
{% from 'elasticsearch/template.map.jinja' import ALL_ADDON_SETTINGS %}
{% endif %}
include:
- ca
@@ -21,6 +17,9 @@ include:
- elasticsearch.ssl
- elasticsearch.config
- elasticsearch.sostatus
{%- if GLOBALS.role != 'so-searchode' %}
- elasticsearch.cluster
{%- endif%}
so-elasticsearch:
docker_container.running:
@@ -108,150 +107,19 @@ delete_so-elasticsearch_so-status.disabled:
- name: /opt/so/conf/so-status/so-status.conf
- regex: ^so-elasticsearch$
{% if GLOBALS.role != "so-searchnode" %}
escomponenttemplates:
file.recurse:
- name: /opt/so/conf/elasticsearch/templates/component
- source: salt://elasticsearch/templates/component
- user: 930
- group: 939
- clean: True
- onchanges_in:
- file: so-elasticsearch-templates-reload
- show_changes: False
# Clean up legacy and non-SO managed templates from the elasticsearch/templates/index/ directory
so_index_template_dir:
file.directory:
- name: /opt/so/conf/elasticsearch/templates/index
- clean: True
{%- if SO_MANAGED_INDICES %}
- require:
{%- for index in SO_MANAGED_INDICES %}
- file: so_index_template_{{index}}
{%- endfor %}
{%- endif %}
# Auto-generate index templates for SO managed indices (directly defined in elasticsearch/defaults.yaml)
# These index templates are for the core SO datasets and are always required
{% for index, settings in ES_INDEX_SETTINGS.items() %}
{% if settings.index_template is defined %}
so_index_template_{{index}}:
file.managed:
- name: /opt/so/conf/elasticsearch/templates/index/{{ index }}-template.json
- source: salt://elasticsearch/base-template.json.jinja
- defaults:
TEMPLATE_CONFIG: {{ settings.index_template }}
- template: jinja
- onchanges_in:
- file: so-elasticsearch-templates-reload
{% endif %}
{% endfor %}
{% if GLOBALS.role != "so-heavynode" %}
# Auto-generate optional index templates for integration | input | content packages
# These index templates are not used by default (until user adds package to an agent policy).
# Pre-configured with standard defaults, and incorporated into SOC configuration for user customization.
{% for index,settings in ALL_ADDON_SETTINGS.items() %}
{% if settings.index_template is defined %}
addon_index_template_{{index}}:
file.managed:
- name: /opt/so/conf/elasticsearch/templates/addon-index/{{ index }}-template.json
- source: salt://elasticsearch/base-template.json.jinja
- defaults:
TEMPLATE_CONFIG: {{ settings.index_template }}
- template: jinja
- show_changes: False
- onchanges_in:
- file: addon-elasticsearch-templates-reload
{% endif %}
{% endfor %}
{% endif %}
{% if GLOBALS.role in GLOBALS.manager_roles %}
so-es-cluster-settings:
cmd.run:
- name: /usr/sbin/so-elasticsearch-cluster-settings
- cwd: /opt/so
- template: jinja
wait_for_so-elasticsearch:
http.wait_for_successful_query:
- name: "https://localhost:9200/"
- username: 'so_elastic'
- password: '{{ ELASTICSEARCHMERGED.auth.users.so_elastic_user.pass }}'
- ssl: True
- verify_ssl: False
- status: 200
- wait_for: 300
- request_interval: 15
- backend: requests
- require:
- docker_container: so-elasticsearch
- file: elasticsearch_sbin_jinja
{% endif %}
# heavynodes will only load ILM policies for SO managed indices. (Indicies defined in elasticsearch/defaults.yaml)
so-elasticsearch-ilm-policy-load:
cmd.run:
- name: /usr/sbin/so-elasticsearch-ilm-policy-load
- cwd: /opt/so
- require:
- docker_container: so-elasticsearch
- file: so-elasticsearch-ilm-policy-load-script
- onchanges:
- file: so-elasticsearch-ilm-policy-load-script
so-elasticsearch-templates-reload:
file.absent:
- name: /opt/so/state/estemplates.txt
addon-elasticsearch-templates-reload:
file.absent:
- name: /opt/so/state/addon_estemplates.txt
# so-elasticsearch-templates-load will have its first successful run during the 'so-elastic-fleet-setup' script
so-elasticsearch-templates:
cmd.run:
{%- if GLOBALS.role == "so-heavynode" %}
- name: /usr/sbin/so-elasticsearch-templates-load --heavynode
{%- else %}
- name: /usr/sbin/so-elasticsearch-templates-load
{%- endif %}
- cwd: /opt/so
- template: jinja
- require:
- docker_container: so-elasticsearch
- file: elasticsearch_sbin_jinja
so-elasticsearch-pipelines:
cmd.run:
- name: /usr/sbin/so-elasticsearch-pipelines {{ GLOBALS.hostname }}
- require:
- docker_container: so-elasticsearch
- file: so-elasticsearch-pipelines-script
so-elasticsearch-roles-load:
cmd.run:
- name: /usr/sbin/so-elasticsearch-roles-load
- cwd: /opt/so
- template: jinja
- require:
- docker_container: so-elasticsearch
- file: elasticsearch_sbin_jinja
{% if grains.role in ['so-managersearch', 'so-manager', 'so-managerhype'] %}
{% set ap = "absent" %}
{% endif %}
{% if grains.role in ['so-eval', 'so-standalone', 'so-heavynode'] %}
{% if ELASTICSEARCHMERGED.index_clean %}
{% set ap = "present" %}
{% else %}
{% set ap = "absent" %}
{% endif %}
{% endif %}
{% if grains.role in ['so-eval', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-manager'] %}
so-elasticsearch-indices-delete:
cron.{{ap}}:
- name: /usr/sbin/so-elasticsearch-indices-delete > /opt/so/log/elasticsearch/cron-elasticsearch-indices-delete.log 2>&1
- identifier: so-elasticsearch-indices-delete
- user: root
- minute: '*/5'
- hour: '*'
- daymonth: '*'
- month: '*'
- dayweek: '*'
{% endif %}
{% endif %}
{% else %}
salt/firewall/map.jinja
+2 -1
View File
@@ -1,5 +1,6 @@
{% from 'vars/globals.map.jinja' import GLOBALS %}
{% from 'docker/docker.map.jinja' import DOCKERMERGED %}
{% from 'telegraf/map.jinja' import TELEGRAFMERGED %}
{% import_yaml 'firewall/defaults.yaml' as FIREWALL_DEFAULT %}
{# add our ip to self #}
@@ -56,7 +57,7 @@
{% endif %}
{# Open Postgres (5432) to minion hostgroups when Telegraf is configured to write to Postgres #}
{% set TG_OUT = (GLOBALS.telegraf_output | default('INFLUXDB')) | upper %}
{% set TG_OUT = TELEGRAFMERGED.output | upper %}
{% if TG_OUT in ['POSTGRES', 'BOTH'] %}
{% if role.startswith('manager') or role == 'standalone' or role == 'eval' %}
{% for r in ['sensor', 'searchnode', 'heavynode', 'receiver', 'fleet', 'idh', 'desktop', 'import'] %}
salt/global/defaults.yaml
+1 -2
View File
@@ -1,4 +1,3 @@
global:
pcapengine: SURICATA
pipeline: REDIS
telegraf_output: BOTH
pipeline: REDIS
salt/global/soc_global.yaml
-8
View File
@@ -59,13 +59,5 @@ global:
description: Allows use of Endgame with Security Onion. This feature requires a license from Endgame.
global: True
advanced: True
telegraf_output:
description: Selects the backend(s) Telegraf writes metrics to. INFLUXDB keeps the current behavior; POSTGRES writes to the grid's Postgres instance; BOTH dual-writes for migration validation.
options:
- INFLUXDB
- POSTGRES
- BOTH
global: True
advanced: True
helpLink: influxdb
salt/manager/tools/sbin/soup
+73 -51
View File
@@ -362,9 +362,8 @@ preupgrade_changes() {
# This function is to add any new pillar items if needed.
echo "Checking to see if changes are needed."
[[ "$INSTALLEDVERSION" =~ ^2\.4\.21[0-9]+$ ]] && up_to_3.0.0
[[ "$INSTALLEDVERSION" == 3.0.0 ]] && up_to_3.1.0
[[ "$INSTALLEDVERSION" =~ ^2\.4\.21[0-9]+$ ]] && up_to_3.0.0
[[ "$INSTALLEDVERSION" == "3.0.0" ]] && up_to_3.1.0
true
}
@@ -373,8 +372,7 @@ postupgrade_changes() {
echo "Running post upgrade processes."
[[ "$POSTVERSION" =~ ^2\.4\.21[0-9]+$ ]] && post_to_3.0.0
[[ "$POSTVERSION" =~ 3.0.0 ]] && post_to_3.1.0
[[ "$POSTVERSION" == "3.0.0" ]] && post_to_3.1.0
true
}
@@ -385,7 +383,7 @@ check_minimum_version() {
fi
}
### 3.0.0 Start ###
### 3.0.0 Scripts ###
convert_suricata_yes_no() {
echo "Starting suricata yes/no values to true/false conversion."
@@ -449,7 +447,6 @@ migrate_pcap_to_suricata() {
}
up_to_3.0.0() {
determine_elastic_agent_upgrade
migrate_pcap_to_suricata
INSTALLEDVERSION=3.0.0
@@ -473,26 +470,58 @@ post_to_3.0.0() {
### 3.0.0 End ###
### 3.1.0 Start ###
### 3.1.0 Scripts ###
elasticsearch_backup_index_templates() {
echo "Backing up current elasticsearch index templates in /opt/so/conf/elasticsearch/templates/index/ to /nsm/backup/3.0.0_elasticsearch_index_templates.tar.gz"
tar -czf /nsm/backup/3.0.0_elasticsearch_index_templates.tar.gz -C /opt/so/conf/elasticsearch/templates/index/ .
}
up_to_3.1.0() {
echo "Nothing to do"
determine_elastic_agent_upgrade
elasticsearch_backup_index_templates
# Clear existing component template state file.
rm -f /opt/so/state/esfleet_component_templates.json
INSTALLEDVERSION=3.1.0
}
post_to_3.1.0() {
# Provision per-minion Telegraf Postgres users for every minion known to the
# manager. postgres.auth iterates manage.up to generate any missing passwords;
# postgres.telegraf_users reconciles the roles and schemas inside the so-postgres
# container. Then push a telegraf state to every minion so their telegraf.conf
# picks up the new credentials on the first apply after soup.
/usr/sbin/so-kibana-space-defaults
# One-time backfill for minions that existed before the postgres Telegraf
# feature shipped. Generate the aggregate pillar on the manager and create
# the per-minion DB roles, then fan each minion's cred into its own pillar
# file. Going forward the reactor handles each new salt-key accept with a
# targeted fan-out, so a manager highstate no longer needs to iterate.
echo "Provisioning Telegraf Postgres users for existing minions."
salt-call --local state.apply postgres.auth postgres.telegraf_users || true
salt-call --local state.apply postgres.auth,postgres.telegraf_users queue=True || true
AGGREGATE_PILLAR=/opt/so/saltstack/local/pillar/postgres/auth.sls
MINIONS_DIR=/opt/so/saltstack/local/pillar/minions
if [[ -f "$AGGREGATE_PILLAR" && -d "$MINIONS_DIR" ]]; then
for pillar_file in "$MINIONS_DIR"/*.sls; do
[[ -f "$pillar_file" ]] || continue
mid=$(basename "$pillar_file" .sls)
[[ "$mid" == adv_* ]] && continue
safe=$(echo "$mid" | tr '.-' '__' | tr '[:upper:]' '[:lower:]')
existing_user=$(so-yaml.py get -r "$pillar_file" postgres.telegraf.user 2>/dev/null || true)
[[ "$existing_user" == "so_telegraf_${safe}" ]] && continue
user=$(so-yaml.py get -r "$AGGREGATE_PILLAR" "postgres.auth.users.telegraf_${safe}.user" 2>/dev/null || true)
pass=$(so-yaml.py get -r "$AGGREGATE_PILLAR" "postgres.auth.users.telegraf_${safe}.pass" 2>/dev/null || true)
[[ -z "$user" || -z "$pass" ]] && continue
so-yaml.py replace "$pillar_file" postgres.telegraf.user "$user" >/dev/null
so-yaml.py replace "$pillar_file" postgres.telegraf.pass "$pass" >/dev/null
done
fi
POSTVERSION=3.1.0
}
### 3.1.0 End ###
repo_sync() {
echo "Sync the local repo."
su socore -c '/usr/sbin/so-repo-sync' || fail "Unable to complete so-repo-sync."
@@ -752,12 +781,12 @@ verify_es_version_compatibility() {
local is_active_intermediate_upgrade=1
# supported upgrade paths for SO-ES versions
declare -A es_upgrade_map=(
["8.18.8"]="9.0.8"
["9.0.8"]="9.3.3"
)
# Elasticsearch MUST upgrade through these versions
declare -A es_to_so_version=(
["8.18.8"]="2.4.190-20251024"
["9.0.8"]="3.0.0-20260331"
)
# Get current Elasticsearch version
@@ -769,26 +798,17 @@ verify_es_version_compatibility() {
exit 160
fi
if ! target_es_version_raw=$(so-yaml.py get $UPDATE_DIR/salt/elasticsearch/defaults.yaml elasticsearch.version); then
# so-yaml.py failed to get the ES version from upgrade versions elasticsearch/defaults.yaml file. Likely they are upgrading to an SO version older than 2.4.110 prior to the ES version pinning and should be OKAY to continue with the upgrade.
if ! target_es_version=$(so-yaml.py get -r $UPDATE_DIR/salt/elasticsearch/defaults.yaml elasticsearch.version); then
echo "Couldn't determine the target Elasticsearch version (post soup version) to ensure compatibility with current Elasticsearch version. Exiting"
# if so-yaml.py failed to get the ES version AND the version we are upgrading to is newer than 2.4.110 then we should bail
if [[ $(cat $UPDATE_DIR/VERSION | cut -d'.' -f3) > 110 ]]; then
echo "Couldn't determine the target Elasticsearch version (post soup version) to ensure compatibility with current Elasticsearch version. Exiting"
exit 160
fi
# allow upgrade to version < 2.4.110 without checking ES version compatibility
return 0
else
target_es_version=$(sed -n '1p' <<< "$target_es_version_raw")
exit 160
fi
for statefile in "${es_required_version_statefile_base}"-*; do
[[ -f $statefile ]] || continue
local es_required_version_statefile_value=$(cat "$statefile")
local es_required_version_statefile_value
es_required_version_statefile_value=$(cat "$statefile")
if [[ "$es_required_version_statefile_value" == "$target_es_version" ]]; then
echo "Intermediate upgrade to ES $target_es_version is in progress. Skipping Elasticsearch version compatibility check."
@@ -797,7 +817,7 @@ verify_es_version_compatibility() {
fi
# use sort to check if es_required_statefile_value is < the current es_version.
if [[ "$(printf '%s\n' $es_required_version_statefile_value $es_version | sort -V | head -n1)" == "$es_required_version_statefile_value" ]]; then
if [[ "$(printf '%s\n' "$es_required_version_statefile_value" "$es_version" | sort -V | head -n1)" == "$es_required_version_statefile_value" ]]; then
rm -f "$statefile"
continue
fi
@@ -808,8 +828,7 @@ verify_es_version_compatibility() {
echo -e "\n##############################################################################################################################\n"
echo "A previously required intermediate Elasticsearch upgrade was detected. Verifying that all Searchnodes/Heavynodes have successfully upgraded Elasticsearch to $es_required_version_statefile_value before proceeding with soup to avoid potential data loss! This command can take up to an hour to complete."
timeout --foreground 4000 bash "$es_verification_script" "$es_required_version_statefile_value" "$statefile"
if [[ $? -ne 0 ]]; then
if ! timeout --foreground 4000 bash "$es_verification_script" "$es_required_version_statefile_value" "$statefile"; then
echo -e "\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
echo "A previous required intermediate Elasticsearch upgrade to $es_required_version_statefile_value has yet to successfully complete across the grid. Please allow time for all Searchnodes/Heavynodes to have upgraded Elasticsearch to $es_required_version_statefile_value before running soup again to avoid potential data loss!"
@@ -826,6 +845,7 @@ verify_es_version_compatibility() {
return 0
fi
# shellcheck disable=SC2076 # Do not want a regex here eg usage " 8.18.8 9.0.8 " =~ " 9.0.8 "
if [[ " ${es_upgrade_map[$es_version]} " =~ " $target_es_version " || "$es_version" == "$target_es_version" ]]; then
# supported upgrade
return 0
@@ -834,7 +854,7 @@ verify_es_version_compatibility() {
if [[ -z "$compatible_versions" ]]; then
# If current ES version is not explicitly defined in the upgrade map, we know they have an intermediate upgrade to do.
# We default to the lowest ES version defined in es_to_so_version as $first_es_required_version
local first_es_required_version=$(printf '%s\n' "${!es_to_so_version[@]}" | sort -V | head -n1)
first_es_required_version=$(printf '%s\n' "${!es_to_so_version[@]}" | sort -V | head -n1)
next_step_so_version=${es_to_so_version[$first_es_required_version]}
required_es_upgrade_version="$first_es_required_version"
else
@@ -853,7 +873,7 @@ verify_es_version_compatibility() {
if [[ $is_airgap -eq 0 ]]; then
run_airgap_intermediate_upgrade
else
if [[ ! -z $ISOLOC ]]; then
if [[ -n $ISOLOC ]]; then
originally_requested_iso_location="$ISOLOC"
fi
# Make sure ISOLOC is not set. Network installs that used soup -f would have ISOLOC set.
@@ -885,7 +905,8 @@ wait_for_salt_minion_with_restart() {
}
run_airgap_intermediate_upgrade() {
local originally_requested_so_version=$(cat $UPDATE_DIR/VERSION)
local originally_requested_so_version
originally_requested_so_version=$(cat "$UPDATE_DIR/VERSION")
# preserve ISOLOC value, so we can try to use it post intermediate upgrade
local originally_requested_iso_location="$ISOLOC"
@@ -897,7 +918,8 @@ run_airgap_intermediate_upgrade() {
while [[ -z "$next_iso_location" ]] || [[ ! -f "$next_iso_location" && ! -b "$next_iso_location" ]]; do
# List removable devices if any are present
local removable_devices=$(lsblk -no PATH,SIZE,TYPE,MOUNTPOINTS,RM | awk '$NF==1')
local removable_devices
removable_devices=$(lsblk -no PATH,SIZE,TYPE,MOUNTPOINTS,RM | awk '$NF==1')
if [[ -n "$removable_devices" ]]; then
echo "PATH SIZE TYPE MOUNTPOINTS RM"
echo "$removable_devices"
@@ -918,21 +940,21 @@ run_airgap_intermediate_upgrade() {
echo "Using $next_iso_location for required intermediary upgrade."
exec bash <<EOF
ISOLOC=$next_iso_location soup -y && \
ISOLOC=$next_iso_location soup -y && \
ISOLOC="$next_iso_location" soup -y && \
ISOLOC="$next_iso_location" soup -y && \
echo -e "\n##############################################################################################################################\n" && \
echo -e "Verifying Elasticsearch was successfully upgraded to $required_es_upgrade_version across the grid. This part can take a while as Searchnodes/Heavynodes sync up with the Manager! \n\nOnce verification completes the next soup will begin automatically. If verification takes longer than 1 hour it will stop waiting and your grid will remain at $next_step_so_version. Allowing for all Searchnodes/Heavynodes to upgrade Elasticsearch to the required version on their own time.\n" && \
timeout --foreground 4000 bash /tmp/so_intermediate_upgrade_verification.sh $required_es_upgrade_version $es_required_version_statefile && \
timeout --foreground 4000 bash /tmp/so_intermediate_upgrade_verification.sh "$required_es_upgrade_version" "$es_required_version_statefile" && \
echo -e "\n##############################################################################################################################\n" && \
# automatically start the next soup if the original ISO isn't using the same block device we just used
if [[ -n "$originally_requested_iso_location" ]] && [[ "$originally_requested_iso_location" != "$next_iso_location" ]]; then
umount /tmp/soagupdate
ISOLOC=$originally_requested_iso_location soup -y && \
ISOLOC=$originally_requested_iso_location soup -y
ISOLOC="$originally_requested_iso_location" soup -y && \
ISOLOC="$originally_requested_iso_location" soup -y
else
echo "Could not automatically start next soup to $originally_requested_so_version. Soup will now exit here at $(cat /etc/soversion)" && \
@@ -948,29 +970,29 @@ run_network_intermediate_upgrade() {
if [[ -n "$BRANCH" ]]; then
local originally_requested_so_branch="$BRANCH"
else
local originally_requested_so_branch="2.4/main"
local originally_requested_so_branch="3/main"
fi
echo "Starting automated intermediate upgrade to $next_step_so_version."
echo "After completion, the system will automatically attempt to upgrade to the latest version."
echo -e "\n##############################################################################################################################\n"
exec bash << EOF
BRANCH=$next_step_so_version soup -y && \
BRANCH=$next_step_so_version soup -y && \
BRANCH="$next_step_so_version" soup -y && \
BRANCH="$next_step_so_version" soup -y && \
echo -e "\n##############################################################################################################################\n" && \
echo -e "Verifying Elasticsearch was successfully upgraded to $required_es_upgrade_version across the grid. This part can take a while as Searchnodes/Heavynodes sync up with the Manager! \n\nOnce verification completes the next soup will begin automatically. If verification takes longer than 1 hour it will stop waiting and your grid will remain at $next_step_so_version. Allowing for all Searchnodes/Heavynodes to upgrade Elasticsearch to the required version on their own time.\n" && \
timeout --foreground 4000 bash /tmp/so_intermediate_upgrade_verification.sh $required_es_upgrade_version $es_required_version_statefile && \
timeout --foreground 4000 bash /tmp/so_intermediate_upgrade_verification.sh "$required_es_upgrade_version" "$es_required_version_statefile" && \
echo -e "\n##############################################################################################################################\n" && \
if [[ -n "$originally_requested_iso_location" ]]; then
# nonairgap soup that used -f originally, runs intermediate upgrade using network + BRANCH, later coming back to the original ISO for the last soup
ISOLOC=$originally_requested_iso_location soup -y && \
ISOLOC=$originally_requested_iso_location soup -y
ISOLOC="$originally_requested_iso_location" soup -y && \
ISOLOC="$originally_requested_iso_location" soup -y
else
BRANCH=$originally_requested_so_branch soup -y && \
BRANCH=$originally_requested_so_branch soup -y
BRANCH="$originally_requested_so_branch" soup -y && \
BRANCH="$originally_requested_so_branch" soup -y
fi
echo -e "\n##############################################################################################################################\n"
EOF
salt/orch/telegraf_postgres_sync.sls
+16 -14
View File
@@ -3,24 +3,26 @@
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{% set MINION = salt['pillar.get']('minion_id') %}
{% set MANAGER = salt['pillar.get']('setup:manager') or salt['grains.get']('master') %}
# Fired by salt/reactor/telegraf_user_sync.sls when salt-key accepts a new
# minion. Only provisions the per-minion pillar entry and DB role on the
# manager; the minion itself will pick up its telegraf config on its first
# highstate during onboarding, so there's no need to push the telegraf state
# from here.
#
# Target the manager via role grains — same pattern as orch/delete_hypervisor.sls.
# The reactor doesn't know the manager's minion id, and grains.master on the
# runner is a hostname, not a targetable id.
{% set FANOUT_MINION = salt['pillar.get']('postgres_fanout_minion', '') %}
manager_sync_telegraf_pg_users:
salt.state:
- tgt: {{ MANAGER }}
- tgt: 'G@role:so-manager or G@role:so-managerhype or G@role:so-managersearch or G@role:so-standalone or G@role:so-eval'
- tgt_type: compound
- sls:
- postgres.auth
- postgres.telegraf_users
- queue: True
{% if MINION and MINION != MANAGER %}
{{ MINION }}_apply_telegraf:
salt.state:
- tgt: {{ MINION }}
- sls:
- telegraf
- queue: True
- require:
- salt: manager_sync_telegraf_pg_users
{% endif %}
{% if FANOUT_MINION %}
- pillar:
postgres_fanout_minion: {{ FANOUT_MINION }}
{% endif %}
salt/postgres/auth.sls
+32
View File
@@ -49,6 +49,38 @@ postgres_auth_pillar:
pass: "{{ entry.pass }}"
{% endfor %}
- show_changes: False
{# Fan a specific minion's telegraf cred out to its own pillar file. Only
runs when postgres_fanout_minion pillar is provided — otherwise this state
is a no-op. That keeps manager highstates from doing N so-yaml.py forks
when nothing changed. The reactor passes postgres_fanout_minion through
the orch on salt-key accept; soup handles bulk backfill separately. #}
{% set fanout_mid = salt['pillar.get']('postgres_fanout_minion') %}
{% if fanout_mid %}
{%- set safe = fanout_mid | replace('.','_') | replace('-','_') | lower %}
{%- set key = 'telegraf_' ~ safe %}
{%- set entry = telegraf_users.get(key) %}
{%- if entry %}
postgres_telegraf_minion_pillar_{{ safe }}:
cmd.run:
- name: |
set -e
PILLAR_FILE=/opt/so/saltstack/local/pillar/minions/{{ fanout_mid }}.sls
if [ ! -f "$PILLAR_FILE" ]; then
echo '{}' > "$PILLAR_FILE"
chown socore:socore "$PILLAR_FILE" 2>/dev/null || true
chmod 640 "$PILLAR_FILE"
fi
/usr/sbin/so-yaml.py replace "$PILLAR_FILE" postgres.telegraf.user '{{ entry.user }}'
/usr/sbin/so-yaml.py replace "$PILLAR_FILE" postgres.telegraf.pass '{{ entry.pass }}'
- unless: |
[ "$(/usr/sbin/so-yaml.py get -r /opt/so/saltstack/local/pillar/minions/{{ fanout_mid }}.sls postgres.telegraf.user 2>/dev/null)" = '{{ entry.user }}' ]
- require:
- file: postgres_auth_pillar
{%- endif %}
{% endif %}
{% else %}
{{sls}}_state_not_allowed:
salt/postgres/config.sls
+5 -4
View File
@@ -21,7 +21,8 @@ postgressecretsdir:
- user: 939
- group: 939
- mode: 700
- makedirs: True
- require:
- file: postgresconfdir
postgresdatadir:
file.directory:
@@ -42,7 +43,8 @@ postgresinitdir:
- name: /opt/so/conf/postgres/init
- user: 939
- group: 939
- makedirs: True
- require:
- file: postgresconfdir
postgresinitusers:
file.managed:
@@ -65,11 +67,10 @@ postgresconf:
postgreshba:
file.managed:
- name: /opt/so/conf/postgres/pg_hba.conf
- source: salt://postgres/files/pg_hba.conf.jinja
- source: salt://postgres/files/pg_hba.conf
- user: 939
- group: 939
- mode: 640
- template: jinja
postgres_super_secret:
file.managed:
salt/postgres/disabled.sls
+6
View File
@@ -18,6 +18,12 @@ so-postgres_so-status.disabled:
- name: /opt/so/conf/so-status/so-status.conf
- regex: ^so-postgres$
so_postgres_backup:
cron.absent:
- name: /usr/sbin/so-postgres-backup > /dev/null 2>&1
- identifier: so_postgres_backup
- user: root
{% else %}
{{sls}}_state_not_allowed:
salt/postgres/enabled.sls
+11
View File
@@ -89,6 +89,17 @@ delete_so-postgres_so-status.disabled:
- name: /opt/so/conf/so-status/so-status.conf
- regex: ^so-postgres$
so_postgres_backup:
cron.present:
- name: /usr/sbin/so-postgres-backup > /dev/null 2>&1
- identifier: so_postgres_backup
- user: root
- minute: '5'
- hour: '0'
- daymonth: '*'
- month: '*'
- dayweek: '*'
{% else %}
{{sls}}_state_not_allowed:
salt/postgres/files/pg_hba.conf.jinja → salt/postgres/files/pg_hba.conf
+5 -4
View File
@@ -1,7 +1,8 @@
{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
https://securityonion.net/license; you may not use this file except in compliance with the
Elastic License 2.0. #}
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
#
# Managed by Salt — do not edit by hand.
# Client authentication config: only local (Unix socket) connections and TLS-wrapped TCP
# connections are accepted. Plain-text `host ...` lines are intentionally omitted so a
salt/postgres/soc_postgres.yaml
+83 -1
View File
@@ -1,7 +1,89 @@
postgres:
enabled:
description: Whether the PostgreSQL database container is enabled on this grid. Backs the assistant store and the Telegraf metrics database.
forcedType: bool
readonly: True
helpLink: influxdb
telegraf:
retention_days:
description: Number of days of Telegraf metrics to keep in the so_telegraf database. Older partitions are dropped hourly by pg_partman.
forcedType: int
helpLink: postgres
config:
max_connections:
description: Maximum number of concurrent PostgreSQL connections.
forcedType: int
global: True
helpLink: postgres
shared_buffers:
description: Amount of memory PostgreSQL uses for shared buffers (e.g. 256MB, 1GB). Raising this improves read cache hit rate at the cost of system RAM.
global: True
helpLink: postgres
log_min_messages:
description: Minimum severity of server messages written to the PostgreSQL log.
options:
- debug1
- info
- notice
- warning
- error
- log
- fatal
global: True
helpLink: postgres
listen_addresses:
description: Interfaces PostgreSQL listens on. Must remain '*' so clients on the docker bridge network can connect.
global: True
advanced: True
helpLink: influxdb
helpLink: postgres
port:
description: TCP port PostgreSQL listens on inside the container. Firewall rules and container port mapping assume 5432.
forcedType: int
global: True
advanced: True
helpLink: postgres
ssl:
description: Whether PostgreSQL accepts TLS connections. Must remain 'on' — pg_hba.conf requires hostssl for TCP.
global: True
advanced: True
helpLink: postgres
ssl_cert_file:
description: Path (inside the container) to the TLS server certificate. Salt-managed.
global: True
advanced: True
helpLink: postgres
ssl_key_file:
description: Path (inside the container) to the TLS server private key. Salt-managed.
global: True
advanced: True
helpLink: postgres
ssl_ca_file:
description: Path (inside the container) to the CA bundle PostgreSQL uses to verify client certificates. Salt-managed.
global: True
advanced: True
helpLink: postgres
hba_file:
description: Path (inside the container) to the pg_hba.conf authentication file. Salt-managed — edit salt/postgres/files/pg_hba.conf.
global: True
advanced: True
helpLink: postgres
log_destination:
description: Where PostgreSQL writes its server log. 'stderr' routes to the container log stream.
global: True
advanced: True
helpLink: postgres
logging_collector:
description: Whether to run a separate logging collector process. Disabled because the docker log stream already captures stderr.
global: True
advanced: True
helpLink: postgres
shared_preload_libraries:
description: Comma-separated list of extensions loaded at server start. Required for pg_cron which drives pg_partman maintenance — do not remove.
global: True
advanced: True
helpLink: postgres
cron.database_name:
description: Database pg_cron schedules jobs in. Must be so_telegraf so partman maintenance runs in the right database context.
global: True
advanced: True
helpLink: postgres
salt/postgres/telegraf_users.sls
+9 -1
View File
@@ -6,8 +6,16 @@
{% from 'allowed_states.map.jinja' import allowed_states %}
{% if sls.split('.')[0] in allowed_states %}
{% from 'vars/globals.map.jinja' import GLOBALS %}
{% from 'telegraf/map.jinja' import TELEGRAFMERGED %}
{% set TG_OUT = (GLOBALS.telegraf_output | default('INFLUXDB')) | upper %}
{# postgres_wait_ready below requires `docker_container: so-postgres`, which is
declared in postgres.enabled. Include it here so state.apply postgres.telegraf_users
on its own (from the reactor orch or from soup) still has that ID in scope. Salt
de-duplicates the circular include. #}
include:
- postgres.enabled
{% set TG_OUT = TELEGRAFMERGED.output | upper %}
{% if TG_OUT in ['POSTGRES', 'BOTH'] %}
# docker_container.running returns as soon as the container starts, but on
salt/backup/tools/sbin/so-postgres-backup → salt/postgres/tools/sbin/so-postgres-backup
View File
salt/postgres/tools/sbin/so-stats-show
+2 -2
View File
@@ -24,7 +24,7 @@ Shows the most recent CPU, memory, disk, and load metrics for each host
from the so_telegraf Postgres database. Without an argument, reports on
every host that has data. With a host, limits output to that one.
Requires: sudo, so-postgres running, global.telegraf_output set to
Requires: sudo, so-postgres running, telegraf.output set to
POSTGRES or BOTH.
EOF
exit 1
@@ -56,7 +56,7 @@ so_psql() {
}
if ! docker exec so-postgres psql -U postgres -lqt 2>/dev/null | cut -d\| -f1 | grep -qw so_telegraf; then
echo "Database so_telegraf not found. Is global.telegraf_output set to POSTGRES or BOTH?"
echo "Database so_telegraf not found. Is telegraf.output set to POSTGRES or BOTH?"
exit 2
fi
salt/reactor/telegraf_user_sync.sls
+3 -3
View File
@@ -3,15 +3,15 @@
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{# Fires on salt/auth. Only act on accepted keys — ignore pending/reject. #}
{% if data.get('act') == 'accept' and data.get('id') %}
{# Fires on salt/key. Only act on successful key acceptance — not reauth. #}
{% if data.get('act') == 'accept' and data.get('result') == True and data.get('id') %}
{{ data['id'] }}_telegraf_pg_sync:
runner.state.orchestrate:
- args:
- mods: orch.telegraf_postgres_sync
- pillar:
minion_id: {{ data['id'] }}
postgres_fanout_minion: {{ data['id'] }}
{% do salt.log.info('telegraf_user_sync reactor: syncing telegraf PG user for minion %s' % data['id']) %}
salt/salt/master.sls
+1 -1
View File
@@ -67,7 +67,7 @@ reactor_config_telegraf:
- name: /etc/salt/master.d/reactor_telegraf.conf
- contents: |
reactor:
- 'salt/auth':
- 'salt/key':
- /opt/so/saltstack/default/salt/reactor/telegraf_user_sync.sls
- user: root
- group: root
salt/telegraf/defaults.yaml
+1
View File
@@ -1,5 +1,6 @@
telegraf:
enabled: False
output: BOTH
config:
interval: '30s'
metric_batch_size: 1000
salt/telegraf/etc/telegraf.conf
+7 -4
View File
@@ -8,11 +8,14 @@
{%- set ZEEK_ENABLED = salt['pillar.get']('zeek:enabled', True) %}
{%- set MDENGINE = GLOBALS.md_engine %}
{%- set LOGSTASH_ENABLED = LOGSTASH_MERGED.enabled %}
{%- set TG_OUT = GLOBALS.telegraf_output | upper %}
{%- set TG_OUT = TELEGRAFMERGED.output | upper %}
{%- set PG_HOST = GLOBALS.manager_ip %}
{%- set PG_SAFE = GLOBALS.minion_id | replace('.','_') | replace('-','_') | lower %}
{%- set PG_USER = 'so_telegraf_' ~ PG_SAFE %}
{%- set PG_PASS = salt['pillar.get']('postgres:auth:users:telegraf_' ~ PG_SAFE ~ ':pass', '') %}
{#- Per-minion telegraf creds are written into the minion's own pillar file
(/opt/so/saltstack/local/pillar/minions/<id>.sls) by postgres.auth on the
manager. Each minion only sees its own password — the aggregate map in
postgres:auth:users is manager-scoped. #}
{%- set PG_USER = salt['pillar.get']('postgres:telegraf:user', '') %}
{%- set PG_PASS = salt['pillar.get']('postgres:telegraf:pass', '') %}
# Global tags can be specified here in key="value" format.
[global_tags]
role = "{{ GLOBALS.role.split('-') | last }}"
salt/telegraf/soc_telegraf.yaml
+9
View File
@@ -4,6 +4,15 @@ telegraf:
forcedType: bool
advanced: True
helpLink: influxdb
output:
description: Selects the backend(s) Telegraf writes metrics to. INFLUXDB keeps the current behavior; POSTGRES writes to the grid's Postgres instance; BOTH dual-writes for migration validation.
options:
- INFLUXDB
- POSTGRES
- BOTH
global: True
advanced: True
helpLink: influxdb
config:
interval:
description: Data collection interval.
salt/vars/globals.map.jinja
-1
View File
@@ -24,7 +24,6 @@
'md_engine': INIT.PILLAR.global.mdengine,
'pcap_engine': GLOBALMERGED.pcapengine,
'pipeline': GLOBALMERGED.pipeline,
'telegraf_output': GLOBALMERGED.telegraf_output,
'so_version': INIT.PILLAR.global.soversion,
'so_docker_gateway': DOCKERMERGED.gateway,
'so_docker_range': DOCKERMERGED.range,