diff --git a/salt/backup/config_backup.sls b/salt/backup/config_backup.sls index c8e342463..a4297444b 100644 --- a/salt/backup/config_backup.sls +++ b/salt/backup/config_backup.sls @@ -33,22 +33,3 @@ so_config_backup: - month: '*' - dayweek: '*' -postgres_backup_script: - file.managed: - - name: /usr/sbin/so-postgres-backup - - user: root - - group: root - - mode: 755 - - source: salt://backup/tools/sbin/so-postgres-backup - -# Add postgres database backup -so_postgres_backup: - cron.present: - - name: /usr/sbin/so-postgres-backup > /dev/null 2>&1 - - identifier: so_postgres_backup - - user: root - - minute: '5' - - hour: '0' - - daymonth: '*' - - month: '*' - - dayweek: '*' diff --git a/salt/docker/defaults.yaml b/salt/docker/defaults.yaml index 900d2cf53..81ff07190 100644 --- a/salt/docker/defaults.yaml +++ b/salt/docker/defaults.yaml @@ -238,7 +238,7 @@ docker: extra_env: [] ulimits: [] 'so-postgres': - final_octet: 89 + final_octet: 47 port_bindings: - 0.0.0.0:5432:5432 custom_bind_mounts: [] diff --git a/salt/elasticsearch/cluster.sls b/salt/elasticsearch/cluster.sls new file mode 100644 index 000000000..7a8a6675c --- /dev/null +++ b/salt/elasticsearch/cluster.sls @@ -0,0 +1,164 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{% from 'allowed_states.map.jinja' import allowed_states %} +{% if sls.split('.')[0] in allowed_states %} +{% from 'vars/globals.map.jinja' import GLOBALS %} +{% from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %} +{% from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS, SO_MANAGED_INDICES %} +{% if GLOBALS.role != 'so-heavynode' %} +{% from 'elasticsearch/template.map.jinja' import ALL_ADDON_SETTINGS %} +{% endif %} + +escomponenttemplates: + file.recurse: + - name: /opt/so/conf/elasticsearch/templates/component + - source: salt://elasticsearch/templates/component + - user: 930 + - group: 939 + - clean: True + - onchanges_in: + - file: so-elasticsearch-templates-reload + - show_changes: False + +# Clean up legacy and non-SO managed templates from the elasticsearch/templates/index/ directory +so_index_template_dir: + file.directory: + - name: /opt/so/conf/elasticsearch/templates/index + - clean: True + {%- if SO_MANAGED_INDICES %} + - require: + {%- for index in SO_MANAGED_INDICES %} + - file: so_index_template_{{index}} + {%- endfor %} + {%- endif %} + +# Auto-generate index templates for SO managed indices (directly defined in elasticsearch/defaults.yaml) +# These index templates are for the core SO datasets and are always required +{% for index, settings in ES_INDEX_SETTINGS.items() %} +{% if settings.index_template is defined %} +so_index_template_{{index}}: + file.managed: + - name: /opt/so/conf/elasticsearch/templates/index/{{ index }}-template.json + - source: salt://elasticsearch/base-template.json.jinja + - defaults: + TEMPLATE_CONFIG: {{ settings.index_template }} + - template: jinja + - onchanges_in: + - file: so-elasticsearch-templates-reload +{% endif %} +{% endfor %} + +{% if GLOBALS.role != "so-heavynode" %} +# Auto-generate optional index templates for integration | input | content packages +# These index templates are not used by default (until user adds package to an agent policy). +# Pre-configured with standard defaults, and incorporated into SOC configuration for user customization. +{% for index,settings in ALL_ADDON_SETTINGS.items() %} +{% if settings.index_template is defined %} +addon_index_template_{{index}}: + file.managed: + - name: /opt/so/conf/elasticsearch/templates/addon-index/{{ index }}-template.json + - source: salt://elasticsearch/base-template.json.jinja + - defaults: + TEMPLATE_CONFIG: {{ settings.index_template }} + - template: jinja + - show_changes: False + - onchanges_in: + - file: addon-elasticsearch-templates-reload +{% endif %} +{% endfor %} +{% endif %} + +{% if GLOBALS.role in GLOBALS.manager_roles %} +so-es-cluster-settings: + cmd.run: + - name: /usr/sbin/so-elasticsearch-cluster-settings + - cwd: /opt/so + - template: jinja + - require: + - docker_container: so-elasticsearch + - file: elasticsearch_sbin_jinja + - http: wait_for_so-elasticsearch +{% endif %} + +# heavynodes will only load ILM policies for SO managed indices. (Indicies defined in elasticsearch/defaults.yaml) +so-elasticsearch-ilm-policy-load: + cmd.run: + - name: /usr/sbin/so-elasticsearch-ilm-policy-load + - cwd: /opt/so + - require: + - docker_container: so-elasticsearch + - file: so-elasticsearch-ilm-policy-load-script + - onchanges: + - file: so-elasticsearch-ilm-policy-load-script + +so-elasticsearch-templates-reload: + file.absent: + - name: /opt/so/state/estemplates.txt + +addon-elasticsearch-templates-reload: + file.absent: + - name: /opt/so/state/addon_estemplates.txt + +# so-elasticsearch-templates-load will have its first successful run during the 'so-elastic-fleet-setup' script +so-elasticsearch-templates: + cmd.run: +{%- if GLOBALS.role == "so-heavynode" %} + - name: /usr/sbin/so-elasticsearch-templates-load --heavynode +{%- else %} + - name: /usr/sbin/so-elasticsearch-templates-load +{%- endif %} + - cwd: /opt/so + - template: jinja + - require: + - docker_container: so-elasticsearch + - file: elasticsearch_sbin_jinja + +so-elasticsearch-pipelines: + cmd.run: + - name: /usr/sbin/so-elasticsearch-pipelines {{ GLOBALS.hostname }} + - require: + - docker_container: so-elasticsearch + - file: so-elasticsearch-pipelines-script + +so-elasticsearch-roles-load: + cmd.run: + - name: /usr/sbin/so-elasticsearch-roles-load + - cwd: /opt/so + - template: jinja + - require: + - docker_container: so-elasticsearch + - file: elasticsearch_sbin_jinja + +{% if grains.role in ['so-managersearch', 'so-manager', 'so-managerhype'] %} +{% set ap = "absent" %} +{% endif %} +{% if grains.role in ['so-eval', 'so-standalone', 'so-heavynode'] %} +{% if ELASTICSEARCHMERGED.index_clean %} +{% set ap = "present" %} +{% else %} +{% set ap = "absent" %} +{% endif %} +{% endif %} +{% if grains.role in ['so-eval', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-manager'] %} +so-elasticsearch-indices-delete: + cron.{{ap}}: + - name: /usr/sbin/so-elasticsearch-indices-delete > /opt/so/log/elasticsearch/cron-elasticsearch-indices-delete.log 2>&1 + - identifier: so-elasticsearch-indices-delete + - user: root + - minute: '*/5' + - hour: '*' + - daymonth: '*' + - month: '*' + - dayweek: '*' +{% endif %} + +{% else %} + +{{sls}}_state_not_allowed: + test.fail_without_changes: + - name: {{sls}}_state_not_allowed + +{% endif %} diff --git a/salt/elasticsearch/enabled.sls b/salt/elasticsearch/enabled.sls index f4031ee5d..ab12b875e 100644 --- a/salt/elasticsearch/enabled.sls +++ b/salt/elasticsearch/enabled.sls @@ -10,10 +10,6 @@ {% from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_NODES %} {% from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_SEED_HOSTS %} {% from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %} -{% from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS, SO_MANAGED_INDICES %} -{% if GLOBALS.role != 'so-heavynode' %} -{% from 'elasticsearch/template.map.jinja' import ALL_ADDON_SETTINGS %} -{% endif %} include: - ca @@ -21,6 +17,9 @@ include: - elasticsearch.ssl - elasticsearch.config - elasticsearch.sostatus +{%- if GLOBALS.role != 'so-searchode' %} + - elasticsearch.cluster +{%- endif%} so-elasticsearch: docker_container.running: @@ -108,150 +107,19 @@ delete_so-elasticsearch_so-status.disabled: - name: /opt/so/conf/so-status/so-status.conf - regex: ^so-elasticsearch$ -{% if GLOBALS.role != "so-searchnode" %} -escomponenttemplates: - file.recurse: - - name: /opt/so/conf/elasticsearch/templates/component - - source: salt://elasticsearch/templates/component - - user: 930 - - group: 939 - - clean: True - - onchanges_in: - - file: so-elasticsearch-templates-reload - - show_changes: False - -# Clean up legacy and non-SO managed templates from the elasticsearch/templates/index/ directory -so_index_template_dir: - file.directory: - - name: /opt/so/conf/elasticsearch/templates/index - - clean: True - {%- if SO_MANAGED_INDICES %} - - require: - {%- for index in SO_MANAGED_INDICES %} - - file: so_index_template_{{index}} - {%- endfor %} - {%- endif %} - -# Auto-generate index templates for SO managed indices (directly defined in elasticsearch/defaults.yaml) -# These index templates are for the core SO datasets and are always required -{% for index, settings in ES_INDEX_SETTINGS.items() %} -{% if settings.index_template is defined %} -so_index_template_{{index}}: - file.managed: - - name: /opt/so/conf/elasticsearch/templates/index/{{ index }}-template.json - - source: salt://elasticsearch/base-template.json.jinja - - defaults: - TEMPLATE_CONFIG: {{ settings.index_template }} - - template: jinja - - onchanges_in: - - file: so-elasticsearch-templates-reload -{% endif %} -{% endfor %} - -{% if GLOBALS.role != "so-heavynode" %} -# Auto-generate optional index templates for integration | input | content packages -# These index templates are not used by default (until user adds package to an agent policy). -# Pre-configured with standard defaults, and incorporated into SOC configuration for user customization. -{% for index,settings in ALL_ADDON_SETTINGS.items() %} -{% if settings.index_template is defined %} -addon_index_template_{{index}}: - file.managed: - - name: /opt/so/conf/elasticsearch/templates/addon-index/{{ index }}-template.json - - source: salt://elasticsearch/base-template.json.jinja - - defaults: - TEMPLATE_CONFIG: {{ settings.index_template }} - - template: jinja - - show_changes: False - - onchanges_in: - - file: addon-elasticsearch-templates-reload -{% endif %} -{% endfor %} -{% endif %} - -{% if GLOBALS.role in GLOBALS.manager_roles %} -so-es-cluster-settings: - cmd.run: - - name: /usr/sbin/so-elasticsearch-cluster-settings - - cwd: /opt/so - - template: jinja +wait_for_so-elasticsearch: + http.wait_for_successful_query: + - name: "https://localhost:9200/" + - username: 'so_elastic' + - password: '{{ ELASTICSEARCHMERGED.auth.users.so_elastic_user.pass }}' + - ssl: True + - verify_ssl: False + - status: 200 + - wait_for: 300 + - request_interval: 15 + - backend: requests - require: - docker_container: so-elasticsearch - - file: elasticsearch_sbin_jinja -{% endif %} - -# heavynodes will only load ILM policies for SO managed indices. (Indicies defined in elasticsearch/defaults.yaml) -so-elasticsearch-ilm-policy-load: - cmd.run: - - name: /usr/sbin/so-elasticsearch-ilm-policy-load - - cwd: /opt/so - - require: - - docker_container: so-elasticsearch - - file: so-elasticsearch-ilm-policy-load-script - - onchanges: - - file: so-elasticsearch-ilm-policy-load-script - -so-elasticsearch-templates-reload: - file.absent: - - name: /opt/so/state/estemplates.txt - -addon-elasticsearch-templates-reload: - file.absent: - - name: /opt/so/state/addon_estemplates.txt - -# so-elasticsearch-templates-load will have its first successful run during the 'so-elastic-fleet-setup' script -so-elasticsearch-templates: - cmd.run: -{%- if GLOBALS.role == "so-heavynode" %} - - name: /usr/sbin/so-elasticsearch-templates-load --heavynode -{%- else %} - - name: /usr/sbin/so-elasticsearch-templates-load -{%- endif %} - - cwd: /opt/so - - template: jinja - - require: - - docker_container: so-elasticsearch - - file: elasticsearch_sbin_jinja - -so-elasticsearch-pipelines: - cmd.run: - - name: /usr/sbin/so-elasticsearch-pipelines {{ GLOBALS.hostname }} - - require: - - docker_container: so-elasticsearch - - file: so-elasticsearch-pipelines-script - -so-elasticsearch-roles-load: - cmd.run: - - name: /usr/sbin/so-elasticsearch-roles-load - - cwd: /opt/so - - template: jinja - - require: - - docker_container: so-elasticsearch - - file: elasticsearch_sbin_jinja - -{% if grains.role in ['so-managersearch', 'so-manager', 'so-managerhype'] %} -{% set ap = "absent" %} -{% endif %} -{% if grains.role in ['so-eval', 'so-standalone', 'so-heavynode'] %} -{% if ELASTICSEARCHMERGED.index_clean %} -{% set ap = "present" %} -{% else %} -{% set ap = "absent" %} -{% endif %} -{% endif %} -{% if grains.role in ['so-eval', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-manager'] %} -so-elasticsearch-indices-delete: - cron.{{ap}}: - - name: /usr/sbin/so-elasticsearch-indices-delete > /opt/so/log/elasticsearch/cron-elasticsearch-indices-delete.log 2>&1 - - identifier: so-elasticsearch-indices-delete - - user: root - - minute: '*/5' - - hour: '*' - - daymonth: '*' - - month: '*' - - dayweek: '*' -{% endif %} - -{% endif %} {% else %} diff --git a/salt/firewall/map.jinja b/salt/firewall/map.jinja index 2821f62b4..61f8215b8 100644 --- a/salt/firewall/map.jinja +++ b/salt/firewall/map.jinja @@ -1,5 +1,6 @@ {% from 'vars/globals.map.jinja' import GLOBALS %} {% from 'docker/docker.map.jinja' import DOCKERMERGED %} +{% from 'telegraf/map.jinja' import TELEGRAFMERGED %} {% import_yaml 'firewall/defaults.yaml' as FIREWALL_DEFAULT %} {# add our ip to self #} @@ -56,7 +57,7 @@ {% endif %} {# Open Postgres (5432) to minion hostgroups when Telegraf is configured to write to Postgres #} -{% set TG_OUT = (GLOBALS.telegraf_output | default('INFLUXDB')) | upper %} +{% set TG_OUT = TELEGRAFMERGED.output | upper %} {% if TG_OUT in ['POSTGRES', 'BOTH'] %} {% if role.startswith('manager') or role == 'standalone' or role == 'eval' %} {% for r in ['sensor', 'searchnode', 'heavynode', 'receiver', 'fleet', 'idh', 'desktop', 'import'] %} diff --git a/salt/global/defaults.yaml b/salt/global/defaults.yaml index d041306a7..92b9c1c1a 100644 --- a/salt/global/defaults.yaml +++ b/salt/global/defaults.yaml @@ -1,4 +1,3 @@ global: pcapengine: SURICATA - pipeline: REDIS - telegraf_output: BOTH \ No newline at end of file + pipeline: REDIS \ No newline at end of file diff --git a/salt/global/soc_global.yaml b/salt/global/soc_global.yaml index 61646168f..31d9f8d3b 100644 --- a/salt/global/soc_global.yaml +++ b/salt/global/soc_global.yaml @@ -59,13 +59,5 @@ global: description: Allows use of Endgame with Security Onion. This feature requires a license from Endgame. global: True advanced: True - telegraf_output: - description: Selects the backend(s) Telegraf writes metrics to. INFLUXDB keeps the current behavior; POSTGRES writes to the grid's Postgres instance; BOTH dual-writes for migration validation. - options: - - INFLUXDB - - POSTGRES - - BOTH - global: True - advanced: True helpLink: influxdb diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 5ed66134f..c19fe487e 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -362,9 +362,8 @@ preupgrade_changes() { # This function is to add any new pillar items if needed. echo "Checking to see if changes are needed." - [[ "$INSTALLEDVERSION" =~ ^2\.4\.21[0-9]+$ ]] && up_to_3.0.0 - [[ "$INSTALLEDVERSION" == 3.0.0 ]] && up_to_3.1.0 - + [[ "$INSTALLEDVERSION" =~ ^2\.4\.21[0-9]+$ ]] && up_to_3.0.0 + [[ "$INSTALLEDVERSION" == "3.0.0" ]] && up_to_3.1.0 true } @@ -373,8 +372,7 @@ postupgrade_changes() { echo "Running post upgrade processes." [[ "$POSTVERSION" =~ ^2\.4\.21[0-9]+$ ]] && post_to_3.0.0 - [[ "$POSTVERSION" =~ 3.0.0 ]] && post_to_3.1.0 - + [[ "$POSTVERSION" == "3.0.0" ]] && post_to_3.1.0 true } @@ -385,7 +383,7 @@ check_minimum_version() { fi } -### 3.0.0 Start ### +### 3.0.0 Scripts ### convert_suricata_yes_no() { echo "Starting suricata yes/no values to true/false conversion." @@ -449,7 +447,6 @@ migrate_pcap_to_suricata() { } up_to_3.0.0() { - determine_elastic_agent_upgrade migrate_pcap_to_suricata INSTALLEDVERSION=3.0.0 @@ -473,26 +470,58 @@ post_to_3.0.0() { ### 3.0.0 End ### -### 3.1.0 Start ### +### 3.1.0 Scripts ### + +elasticsearch_backup_index_templates() { + echo "Backing up current elasticsearch index templates in /opt/so/conf/elasticsearch/templates/index/ to /nsm/backup/3.0.0_elasticsearch_index_templates.tar.gz" + tar -czf /nsm/backup/3.0.0_elasticsearch_index_templates.tar.gz -C /opt/so/conf/elasticsearch/templates/index/ . +} + up_to_3.1.0() { - echo "Nothing to do" + determine_elastic_agent_upgrade + elasticsearch_backup_index_templates + # Clear existing component template state file. + rm -f /opt/so/state/esfleet_component_templates.json + + INSTALLEDVERSION=3.1.0 } post_to_3.1.0() { - # Provision per-minion Telegraf Postgres users for every minion known to the - # manager. postgres.auth iterates manage.up to generate any missing passwords; - # postgres.telegraf_users reconciles the roles and schemas inside the so-postgres - # container. Then push a telegraf state to every minion so their telegraf.conf - # picks up the new credentials on the first apply after soup. + /usr/sbin/so-kibana-space-defaults + + # One-time backfill for minions that existed before the postgres Telegraf + # feature shipped. Generate the aggregate pillar on the manager and create + # the per-minion DB roles, then fan each minion's cred into its own pillar + # file. Going forward the reactor handles each new salt-key accept with a + # targeted fan-out, so a manager highstate no longer needs to iterate. echo "Provisioning Telegraf Postgres users for existing minions." - salt-call --local state.apply postgres.auth postgres.telegraf_users || true + salt-call --local state.apply postgres.auth,postgres.telegraf_users queue=True || true + + AGGREGATE_PILLAR=/opt/so/saltstack/local/pillar/postgres/auth.sls + MINIONS_DIR=/opt/so/saltstack/local/pillar/minions + if [[ -f "$AGGREGATE_PILLAR" && -d "$MINIONS_DIR" ]]; then + for pillar_file in "$MINIONS_DIR"/*.sls; do + [[ -f "$pillar_file" ]] || continue + mid=$(basename "$pillar_file" .sls) + [[ "$mid" == adv_* ]] && continue + safe=$(echo "$mid" | tr '.-' '__' | tr '[:upper:]' '[:lower:]') + existing_user=$(so-yaml.py get -r "$pillar_file" postgres.telegraf.user 2>/dev/null || true) + [[ "$existing_user" == "so_telegraf_${safe}" ]] && continue + user=$(so-yaml.py get -r "$AGGREGATE_PILLAR" "postgres.auth.users.telegraf_${safe}.user" 2>/dev/null || true) + pass=$(so-yaml.py get -r "$AGGREGATE_PILLAR" "postgres.auth.users.telegraf_${safe}.pass" 2>/dev/null || true) + [[ -z "$user" || -z "$pass" ]] && continue + so-yaml.py replace "$pillar_file" postgres.telegraf.user "$user" >/dev/null + so-yaml.py replace "$pillar_file" postgres.telegraf.pass "$pass" >/dev/null + done + fi POSTVERSION=3.1.0 } ### 3.1.0 End ### + repo_sync() { echo "Sync the local repo." su socore -c '/usr/sbin/so-repo-sync' || fail "Unable to complete so-repo-sync." @@ -752,12 +781,12 @@ verify_es_version_compatibility() { local is_active_intermediate_upgrade=1 # supported upgrade paths for SO-ES versions declare -A es_upgrade_map=( - ["8.18.8"]="9.0.8" + ["9.0.8"]="9.3.3" ) # Elasticsearch MUST upgrade through these versions declare -A es_to_so_version=( - ["8.18.8"]="2.4.190-20251024" + ["9.0.8"]="3.0.0-20260331" ) # Get current Elasticsearch version @@ -769,26 +798,17 @@ verify_es_version_compatibility() { exit 160 fi - if ! target_es_version_raw=$(so-yaml.py get $UPDATE_DIR/salt/elasticsearch/defaults.yaml elasticsearch.version); then - # so-yaml.py failed to get the ES version from upgrade versions elasticsearch/defaults.yaml file. Likely they are upgrading to an SO version older than 2.4.110 prior to the ES version pinning and should be OKAY to continue with the upgrade. + if ! target_es_version=$(so-yaml.py get -r $UPDATE_DIR/salt/elasticsearch/defaults.yaml elasticsearch.version); then + echo "Couldn't determine the target Elasticsearch version (post soup version) to ensure compatibility with current Elasticsearch version. Exiting" - # if so-yaml.py failed to get the ES version AND the version we are upgrading to is newer than 2.4.110 then we should bail - if [[ $(cat $UPDATE_DIR/VERSION | cut -d'.' -f3) > 110 ]]; then - echo "Couldn't determine the target Elasticsearch version (post soup version) to ensure compatibility with current Elasticsearch version. Exiting" - - exit 160 - fi - - # allow upgrade to version < 2.4.110 without checking ES version compatibility - return 0 - else - target_es_version=$(sed -n '1p' <<< "$target_es_version_raw") + exit 160 fi for statefile in "${es_required_version_statefile_base}"-*; do [[ -f $statefile ]] || continue - local es_required_version_statefile_value=$(cat "$statefile") + local es_required_version_statefile_value + es_required_version_statefile_value=$(cat "$statefile") if [[ "$es_required_version_statefile_value" == "$target_es_version" ]]; then echo "Intermediate upgrade to ES $target_es_version is in progress. Skipping Elasticsearch version compatibility check." @@ -797,7 +817,7 @@ verify_es_version_compatibility() { fi # use sort to check if es_required_statefile_value is < the current es_version. - if [[ "$(printf '%s\n' $es_required_version_statefile_value $es_version | sort -V | head -n1)" == "$es_required_version_statefile_value" ]]; then + if [[ "$(printf '%s\n' "$es_required_version_statefile_value" "$es_version" | sort -V | head -n1)" == "$es_required_version_statefile_value" ]]; then rm -f "$statefile" continue fi @@ -808,8 +828,7 @@ verify_es_version_compatibility() { echo -e "\n##############################################################################################################################\n" echo "A previously required intermediate Elasticsearch upgrade was detected. Verifying that all Searchnodes/Heavynodes have successfully upgraded Elasticsearch to $es_required_version_statefile_value before proceeding with soup to avoid potential data loss! This command can take up to an hour to complete." - timeout --foreground 4000 bash "$es_verification_script" "$es_required_version_statefile_value" "$statefile" - if [[ $? -ne 0 ]]; then + if ! timeout --foreground 4000 bash "$es_verification_script" "$es_required_version_statefile_value" "$statefile"; then echo -e "\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n" echo "A previous required intermediate Elasticsearch upgrade to $es_required_version_statefile_value has yet to successfully complete across the grid. Please allow time for all Searchnodes/Heavynodes to have upgraded Elasticsearch to $es_required_version_statefile_value before running soup again to avoid potential data loss!" @@ -826,6 +845,7 @@ verify_es_version_compatibility() { return 0 fi + # shellcheck disable=SC2076 # Do not want a regex here eg usage " 8.18.8 9.0.8 " =~ " 9.0.8 " if [[ " ${es_upgrade_map[$es_version]} " =~ " $target_es_version " || "$es_version" == "$target_es_version" ]]; then # supported upgrade return 0 @@ -834,7 +854,7 @@ verify_es_version_compatibility() { if [[ -z "$compatible_versions" ]]; then # If current ES version is not explicitly defined in the upgrade map, we know they have an intermediate upgrade to do. # We default to the lowest ES version defined in es_to_so_version as $first_es_required_version - local first_es_required_version=$(printf '%s\n' "${!es_to_so_version[@]}" | sort -V | head -n1) + first_es_required_version=$(printf '%s\n' "${!es_to_so_version[@]}" | sort -V | head -n1) next_step_so_version=${es_to_so_version[$first_es_required_version]} required_es_upgrade_version="$first_es_required_version" else @@ -853,7 +873,7 @@ verify_es_version_compatibility() { if [[ $is_airgap -eq 0 ]]; then run_airgap_intermediate_upgrade else - if [[ ! -z $ISOLOC ]]; then + if [[ -n $ISOLOC ]]; then originally_requested_iso_location="$ISOLOC" fi # Make sure ISOLOC is not set. Network installs that used soup -f would have ISOLOC set. @@ -885,7 +905,8 @@ wait_for_salt_minion_with_restart() { } run_airgap_intermediate_upgrade() { - local originally_requested_so_version=$(cat $UPDATE_DIR/VERSION) + local originally_requested_so_version + originally_requested_so_version=$(cat "$UPDATE_DIR/VERSION") # preserve ISOLOC value, so we can try to use it post intermediate upgrade local originally_requested_iso_location="$ISOLOC" @@ -897,7 +918,8 @@ run_airgap_intermediate_upgrade() { while [[ -z "$next_iso_location" ]] || [[ ! -f "$next_iso_location" && ! -b "$next_iso_location" ]]; do # List removable devices if any are present - local removable_devices=$(lsblk -no PATH,SIZE,TYPE,MOUNTPOINTS,RM | awk '$NF==1') + local removable_devices + removable_devices=$(lsblk -no PATH,SIZE,TYPE,MOUNTPOINTS,RM | awk '$NF==1') if [[ -n "$removable_devices" ]]; then echo "PATH SIZE TYPE MOUNTPOINTS RM" echo "$removable_devices" @@ -918,21 +940,21 @@ run_airgap_intermediate_upgrade() { echo "Using $next_iso_location for required intermediary upgrade." exec bash < "$PILLAR_FILE" + chown socore:socore "$PILLAR_FILE" 2>/dev/null || true + chmod 640 "$PILLAR_FILE" + fi + /usr/sbin/so-yaml.py replace "$PILLAR_FILE" postgres.telegraf.user '{{ entry.user }}' + /usr/sbin/so-yaml.py replace "$PILLAR_FILE" postgres.telegraf.pass '{{ entry.pass }}' + - unless: | + [ "$(/usr/sbin/so-yaml.py get -r /opt/so/saltstack/local/pillar/minions/{{ fanout_mid }}.sls postgres.telegraf.user 2>/dev/null)" = '{{ entry.user }}' ] + - require: + - file: postgres_auth_pillar + + {%- endif %} + {% endif %} {% else %} {{sls}}_state_not_allowed: diff --git a/salt/postgres/config.sls b/salt/postgres/config.sls index 76a926d59..11ca52649 100644 --- a/salt/postgres/config.sls +++ b/salt/postgres/config.sls @@ -21,7 +21,8 @@ postgressecretsdir: - user: 939 - group: 939 - mode: 700 - - makedirs: True + - require: + - file: postgresconfdir postgresdatadir: file.directory: @@ -42,7 +43,8 @@ postgresinitdir: - name: /opt/so/conf/postgres/init - user: 939 - group: 939 - - makedirs: True + - require: + - file: postgresconfdir postgresinitusers: file.managed: @@ -65,11 +67,10 @@ postgresconf: postgreshba: file.managed: - name: /opt/so/conf/postgres/pg_hba.conf - - source: salt://postgres/files/pg_hba.conf.jinja + - source: salt://postgres/files/pg_hba.conf - user: 939 - group: 939 - mode: 640 - - template: jinja postgres_super_secret: file.managed: diff --git a/salt/postgres/disabled.sls b/salt/postgres/disabled.sls index 56dc451b7..4b5b62328 100644 --- a/salt/postgres/disabled.sls +++ b/salt/postgres/disabled.sls @@ -18,6 +18,12 @@ so-postgres_so-status.disabled: - name: /opt/so/conf/so-status/so-status.conf - regex: ^so-postgres$ +so_postgres_backup: + cron.absent: + - name: /usr/sbin/so-postgres-backup > /dev/null 2>&1 + - identifier: so_postgres_backup + - user: root + {% else %} {{sls}}_state_not_allowed: diff --git a/salt/postgres/enabled.sls b/salt/postgres/enabled.sls index 4c5838466..b3abb621e 100644 --- a/salt/postgres/enabled.sls +++ b/salt/postgres/enabled.sls @@ -89,6 +89,17 @@ delete_so-postgres_so-status.disabled: - name: /opt/so/conf/so-status/so-status.conf - regex: ^so-postgres$ +so_postgres_backup: + cron.present: + - name: /usr/sbin/so-postgres-backup > /dev/null 2>&1 + - identifier: so_postgres_backup + - user: root + - minute: '5' + - hour: '0' + - daymonth: '*' + - month: '*' + - dayweek: '*' + {% else %} {{sls}}_state_not_allowed: diff --git a/salt/postgres/files/pg_hba.conf.jinja b/salt/postgres/files/pg_hba.conf similarity index 67% rename from salt/postgres/files/pg_hba.conf.jinja rename to salt/postgres/files/pg_hba.conf index 1d6a22a04..e7d31c05f 100644 --- a/salt/postgres/files/pg_hba.conf.jinja +++ b/salt/postgres/files/pg_hba.conf @@ -1,7 +1,8 @@ -{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one - or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at - https://securityonion.net/license; you may not use this file except in compliance with the - Elastic License 2.0. #} +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. +# # Managed by Salt — do not edit by hand. # Client authentication config: only local (Unix socket) connections and TLS-wrapped TCP # connections are accepted. Plain-text `host ...` lines are intentionally omitted so a diff --git a/salt/postgres/soc_postgres.yaml b/salt/postgres/soc_postgres.yaml index 8b4e22921..4b25cd4f5 100644 --- a/salt/postgres/soc_postgres.yaml +++ b/salt/postgres/soc_postgres.yaml @@ -1,7 +1,89 @@ postgres: + enabled: + description: Whether the PostgreSQL database container is enabled on this grid. Backs the assistant store and the Telegraf metrics database. + forcedType: bool + readonly: True + helpLink: influxdb telegraf: retention_days: description: Number of days of Telegraf metrics to keep in the so_telegraf database. Older partitions are dropped hourly by pg_partman. forcedType: int + helpLink: postgres + config: + max_connections: + description: Maximum number of concurrent PostgreSQL connections. + forcedType: int + global: True + helpLink: postgres + shared_buffers: + description: Amount of memory PostgreSQL uses for shared buffers (e.g. 256MB, 1GB). Raising this improves read cache hit rate at the cost of system RAM. + global: True + helpLink: postgres + log_min_messages: + description: Minimum severity of server messages written to the PostgreSQL log. + options: + - debug1 + - info + - notice + - warning + - error + - log + - fatal + global: True + helpLink: postgres + listen_addresses: + description: Interfaces PostgreSQL listens on. Must remain '*' so clients on the docker bridge network can connect. + global: True advanced: True - helpLink: influxdb + helpLink: postgres + port: + description: TCP port PostgreSQL listens on inside the container. Firewall rules and container port mapping assume 5432. + forcedType: int + global: True + advanced: True + helpLink: postgres + ssl: + description: Whether PostgreSQL accepts TLS connections. Must remain 'on' — pg_hba.conf requires hostssl for TCP. + global: True + advanced: True + helpLink: postgres + ssl_cert_file: + description: Path (inside the container) to the TLS server certificate. Salt-managed. + global: True + advanced: True + helpLink: postgres + ssl_key_file: + description: Path (inside the container) to the TLS server private key. Salt-managed. + global: True + advanced: True + helpLink: postgres + ssl_ca_file: + description: Path (inside the container) to the CA bundle PostgreSQL uses to verify client certificates. Salt-managed. + global: True + advanced: True + helpLink: postgres + hba_file: + description: Path (inside the container) to the pg_hba.conf authentication file. Salt-managed — edit salt/postgres/files/pg_hba.conf. + global: True + advanced: True + helpLink: postgres + log_destination: + description: Where PostgreSQL writes its server log. 'stderr' routes to the container log stream. + global: True + advanced: True + helpLink: postgres + logging_collector: + description: Whether to run a separate logging collector process. Disabled because the docker log stream already captures stderr. + global: True + advanced: True + helpLink: postgres + shared_preload_libraries: + description: Comma-separated list of extensions loaded at server start. Required for pg_cron which drives pg_partman maintenance — do not remove. + global: True + advanced: True + helpLink: postgres + cron.database_name: + description: Database pg_cron schedules jobs in. Must be so_telegraf so partman maintenance runs in the right database context. + global: True + advanced: True + helpLink: postgres diff --git a/salt/postgres/telegraf_users.sls b/salt/postgres/telegraf_users.sls index cab65d8a8..dbbc0f03e 100644 --- a/salt/postgres/telegraf_users.sls +++ b/salt/postgres/telegraf_users.sls @@ -6,8 +6,16 @@ {% from 'allowed_states.map.jinja' import allowed_states %} {% if sls.split('.')[0] in allowed_states %} {% from 'vars/globals.map.jinja' import GLOBALS %} +{% from 'telegraf/map.jinja' import TELEGRAFMERGED %} -{% set TG_OUT = (GLOBALS.telegraf_output | default('INFLUXDB')) | upper %} +{# postgres_wait_ready below requires `docker_container: so-postgres`, which is + declared in postgres.enabled. Include it here so state.apply postgres.telegraf_users + on its own (from the reactor orch or from soup) still has that ID in scope. Salt + de-duplicates the circular include. #} +include: + - postgres.enabled + +{% set TG_OUT = TELEGRAFMERGED.output | upper %} {% if TG_OUT in ['POSTGRES', 'BOTH'] %} # docker_container.running returns as soon as the container starts, but on diff --git a/salt/backup/tools/sbin/so-postgres-backup b/salt/postgres/tools/sbin/so-postgres-backup similarity index 100% rename from salt/backup/tools/sbin/so-postgres-backup rename to salt/postgres/tools/sbin/so-postgres-backup diff --git a/salt/postgres/tools/sbin/so-stats-show b/salt/postgres/tools/sbin/so-stats-show index 102b51ccd..3cf7a05d8 100644 --- a/salt/postgres/tools/sbin/so-stats-show +++ b/salt/postgres/tools/sbin/so-stats-show @@ -24,7 +24,7 @@ Shows the most recent CPU, memory, disk, and load metrics for each host from the so_telegraf Postgres database. Without an argument, reports on every host that has data. With a host, limits output to that one. -Requires: sudo, so-postgres running, global.telegraf_output set to +Requires: sudo, so-postgres running, telegraf.output set to POSTGRES or BOTH. EOF exit 1 @@ -56,7 +56,7 @@ so_psql() { } if ! docker exec so-postgres psql -U postgres -lqt 2>/dev/null | cut -d\| -f1 | grep -qw so_telegraf; then - echo "Database so_telegraf not found. Is global.telegraf_output set to POSTGRES or BOTH?" + echo "Database so_telegraf not found. Is telegraf.output set to POSTGRES or BOTH?" exit 2 fi diff --git a/salt/reactor/telegraf_user_sync.sls b/salt/reactor/telegraf_user_sync.sls index abf35d3b2..075dbf62e 100644 --- a/salt/reactor/telegraf_user_sync.sls +++ b/salt/reactor/telegraf_user_sync.sls @@ -3,15 +3,15 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -{# Fires on salt/auth. Only act on accepted keys — ignore pending/reject. #} -{% if data.get('act') == 'accept' and data.get('id') %} +{# Fires on salt/key. Only act on successful key acceptance — not reauth. #} +{% if data.get('act') == 'accept' and data.get('result') == True and data.get('id') %} {{ data['id'] }}_telegraf_pg_sync: runner.state.orchestrate: - args: - mods: orch.telegraf_postgres_sync - pillar: - minion_id: {{ data['id'] }} + postgres_fanout_minion: {{ data['id'] }} {% do salt.log.info('telegraf_user_sync reactor: syncing telegraf PG user for minion %s' % data['id']) %} diff --git a/salt/salt/master.sls b/salt/salt/master.sls index 7e3e48074..e61b09d21 100644 --- a/salt/salt/master.sls +++ b/salt/salt/master.sls @@ -67,7 +67,7 @@ reactor_config_telegraf: - name: /etc/salt/master.d/reactor_telegraf.conf - contents: | reactor: - - 'salt/auth': + - 'salt/key': - /opt/so/saltstack/default/salt/reactor/telegraf_user_sync.sls - user: root - group: root diff --git a/salt/telegraf/defaults.yaml b/salt/telegraf/defaults.yaml index ef6c2bc77..ead122b0a 100644 --- a/salt/telegraf/defaults.yaml +++ b/salt/telegraf/defaults.yaml @@ -1,5 +1,6 @@ telegraf: enabled: False + output: BOTH config: interval: '30s' metric_batch_size: 1000 diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf index d28dc7f96..001a61d93 100644 --- a/salt/telegraf/etc/telegraf.conf +++ b/salt/telegraf/etc/telegraf.conf @@ -8,11 +8,14 @@ {%- set ZEEK_ENABLED = salt['pillar.get']('zeek:enabled', True) %} {%- set MDENGINE = GLOBALS.md_engine %} {%- set LOGSTASH_ENABLED = LOGSTASH_MERGED.enabled %} -{%- set TG_OUT = GLOBALS.telegraf_output | upper %} +{%- set TG_OUT = TELEGRAFMERGED.output | upper %} {%- set PG_HOST = GLOBALS.manager_ip %} -{%- set PG_SAFE = GLOBALS.minion_id | replace('.','_') | replace('-','_') | lower %} -{%- set PG_USER = 'so_telegraf_' ~ PG_SAFE %} -{%- set PG_PASS = salt['pillar.get']('postgres:auth:users:telegraf_' ~ PG_SAFE ~ ':pass', '') %} +{#- Per-minion telegraf creds are written into the minion's own pillar file + (/opt/so/saltstack/local/pillar/minions/.sls) by postgres.auth on the + manager. Each minion only sees its own password — the aggregate map in + postgres:auth:users is manager-scoped. #} +{%- set PG_USER = salt['pillar.get']('postgres:telegraf:user', '') %} +{%- set PG_PASS = salt['pillar.get']('postgres:telegraf:pass', '') %} # Global tags can be specified here in key="value" format. [global_tags] role = "{{ GLOBALS.role.split('-') | last }}" diff --git a/salt/telegraf/soc_telegraf.yaml b/salt/telegraf/soc_telegraf.yaml index 40ae7fed8..4b9a2e3d1 100644 --- a/salt/telegraf/soc_telegraf.yaml +++ b/salt/telegraf/soc_telegraf.yaml @@ -4,6 +4,15 @@ telegraf: forcedType: bool advanced: True helpLink: influxdb + output: + description: Selects the backend(s) Telegraf writes metrics to. INFLUXDB keeps the current behavior; POSTGRES writes to the grid's Postgres instance; BOTH dual-writes for migration validation. + options: + - INFLUXDB + - POSTGRES + - BOTH + global: True + advanced: True + helpLink: influxdb config: interval: description: Data collection interval. diff --git a/salt/vars/globals.map.jinja b/salt/vars/globals.map.jinja index 787691b13..385db02ae 100644 --- a/salt/vars/globals.map.jinja +++ b/salt/vars/globals.map.jinja @@ -24,7 +24,6 @@ 'md_engine': INIT.PILLAR.global.mdengine, 'pcap_engine': GLOBALMERGED.pcapengine, 'pipeline': GLOBALMERGED.pipeline, - 'telegraf_output': GLOBALMERGED.telegraf_output, 'so_version': INIT.PILLAR.global.soversion, 'so_docker_gateway': DOCKERMERGED.gateway, 'so_docker_range': DOCKERMERGED.range,