diff --git a/.github/DISCUSSION_TEMPLATE/3-0.yml b/.github/DISCUSSION_TEMPLATE/3-0.yml index 286ac823a..3fb9e5b30 100644 --- a/.github/DISCUSSION_TEMPLATE/3-0.yml +++ b/.github/DISCUSSION_TEMPLATE/3-0.yml @@ -10,6 +10,7 @@ body: options: - - 3.0.0 + - 3.1.0 - Other (please provide detail below) validations: required: true diff --git a/DOWNLOAD_AND_VERIFY_ISO.md b/DOWNLOAD_AND_VERIFY_ISO.md index 47937c1b9..a0ea874fa 100644 --- a/DOWNLOAD_AND_VERIFY_ISO.md +++ b/DOWNLOAD_AND_VERIFY_ISO.md @@ -1,17 +1,17 @@ -### 3.0.0-20260331 ISO image released on 2026/03/31 +### 3.1.0-20260521 ISO image released on 2026/05/21 ### Download and Verify -3.0.0-20260331 ISO image: -https://download.securityonion.net/file/securityonion/securityonion-3.0.0-20260331.iso +3.1.0-20260521 ISO image: +https://download.securityonion.net/file/securityonion/securityonion-3.1.0-20260521.iso -MD5: ECD318A1662A6FDE0EF213F5A9BD4B07 -SHA1: E55BE314440CCF3392DC0B06BC5E270B43176D9C -SHA256: 7FC47405E335CBE5C2B6C51FE7AC60248F35CBE504907B8B5A33822B23F8F4D5 +MD5: A853BC118639ABCE1795D6E313BFFBDE +SHA1: FCA615AD6E31710B33AE5870FEF447861FDB3B8F +SHA256: CE2A5947274D9ED2C5068A1FD46B64C4FEF70445EA9B61A98DD3621781329F2C Signature for ISO image: -https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.0.0-20260331.iso.sig +https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.1.0-20260521.iso.sig Signing key: https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/3/main/KEYS @@ -25,22 +25,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/3/ Download the signature file for the ISO: ``` -wget https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.0.0-20260331.iso.sig +wget https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.1.0-20260521.iso.sig ``` Download the ISO image: ``` -wget https://download.securityonion.net/file/securityonion/securityonion-3.0.0-20260331.iso +wget https://download.securityonion.net/file/securityonion/securityonion-3.1.0-20260521.iso ``` Verify the downloaded ISO image using the signature file: ``` -gpg --verify securityonion-3.0.0-20260331.iso.sig securityonion-3.0.0-20260331.iso +gpg --verify securityonion-3.1.0-20260521.iso.sig securityonion-3.1.0-20260521.iso ``` The output should show "Good signature" and the Primary key fingerprint should match what's shown below: ``` -gpg: Signature made Mon 30 Mar 2026 06:22:14 PM EDT using RSA key ID FE507013 +gpg: Signature made Thu 21 May 2026 11:10:01 AM EDT using RSA key ID FE507013 gpg: Good signature from "Security Onion Solutions, LLC " gpg: WARNING: This key is not certified with a trusted signature! gpg: There is no indication that the signature belongs to the owner. diff --git a/VERSION b/VERSION index 4a36342fc..fd2a01863 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.0.0 +3.1.0 diff --git a/pillar/elasticsearch/index_templates.sls b/pillar/elasticsearch/index_templates.sls deleted file mode 100644 index a02a1818c..000000000 --- a/pillar/elasticsearch/index_templates.sls +++ /dev/null @@ -1,2 +0,0 @@ -elasticsearch: - index_settings: diff --git a/pillar/telegraf/creds.sls b/pillar/telegraf/creds.sls new file mode 100644 index 000000000..8521bfbd9 --- /dev/null +++ b/pillar/telegraf/creds.sls @@ -0,0 +1,12 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +# Per-minion Telegraf Postgres credentials. so-telegraf-cred on the manager is +# the single writer; it mutates /opt/so/saltstack/local/pillar/telegraf/creds.sls +# under flock. Pillar_roots order (local before default) means the populated +# copy shadows this default on any real grid; this file exists so the pillar +# key is always defined on fresh installs and when no minions have creds yet. +telegraf: + postgres_creds: {} diff --git a/pillar/top.sls b/pillar/top.sls index 6cdc4808a..712629dbf 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -17,6 +17,7 @@ base: - sensoroni.adv_sensoroni - telegraf.soc_telegraf - telegraf.adv_telegraf + - telegraf.creds - versionlock.soc_versionlock - versionlock.adv_versionlock - soc.license @@ -38,6 +39,9 @@ base: {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %} - elasticsearch.auth {% endif %} + {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/postgres/auth.sls') %} + - postgres.auth + {% endif %} {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/kibana/secrets.sls') %} - kibana.secrets {% endif %} @@ -60,6 +64,8 @@ base: - redis.adv_redis - influxdb.soc_influxdb - influxdb.adv_influxdb + - postgres.soc_postgres + - postgres.adv_postgres - elasticsearch.nodes - elasticsearch.soc_elasticsearch - elasticsearch.adv_elasticsearch @@ -97,10 +103,12 @@ base: - node_data.ips - secrets - healthcheck.eval - - elasticsearch.index_templates {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %} - elasticsearch.auth {% endif %} + {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/postgres/auth.sls') %} + - postgres.auth + {% endif %} {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/kibana/secrets.sls') %} - kibana.secrets {% endif %} @@ -126,6 +134,8 @@ base: - redis.adv_redis - influxdb.soc_influxdb - influxdb.adv_influxdb + - postgres.soc_postgres + - postgres.adv_postgres - backup.soc_backup - backup.adv_backup - zeek.soc_zeek @@ -142,10 +152,12 @@ base: - logstash.nodes - logstash.soc_logstash - logstash.adv_logstash - - elasticsearch.index_templates {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %} - elasticsearch.auth {% endif %} + {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/postgres/auth.sls') %} + - postgres.auth + {% endif %} {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/kibana/secrets.sls') %} - kibana.secrets {% endif %} @@ -160,6 +172,8 @@ base: - redis.adv_redis - influxdb.soc_influxdb - influxdb.adv_influxdb + - postgres.soc_postgres + - postgres.adv_postgres - elasticsearch.nodes - elasticsearch.soc_elasticsearch - elasticsearch.adv_elasticsearch @@ -256,10 +270,12 @@ base: '*_import': - node_data.ips - secrets - - elasticsearch.index_templates {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %} - elasticsearch.auth {% endif %} + {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/postgres/auth.sls') %} + - postgres.auth + {% endif %} {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/kibana/secrets.sls') %} - kibana.secrets {% endif %} @@ -285,6 +301,8 @@ base: - redis.adv_redis - influxdb.soc_influxdb - influxdb.adv_influxdb + - postgres.soc_postgres + - postgres.adv_postgres - zeek.soc_zeek - zeek.adv_zeek - bpf.soc_bpf diff --git a/salt/allowed_states.map.jinja b/salt/allowed_states.map.jinja index 1fac0f0e3..c831b45fe 100644 --- a/salt/allowed_states.map.jinja +++ b/salt/allowed_states.map.jinja @@ -29,10 +29,14 @@ 'manager', 'nginx', 'influxdb', + 'postgres', + 'postgres.auth', 'soc', 'kratos', 'hydra', 'elasticfleet', + 'elasticfleet.manager', + 'elasticsearch.cluster', 'elastic-fleet-package-registry', 'utility' ] %} @@ -77,7 +81,7 @@ ), 'so-heavynode': ( sensor_states + - ['elasticagent', 'elasticsearch', 'logstash', 'redis', 'nginx'] + ['elasticagent', 'elasticsearch', 'elasticsearch.cluster', 'logstash', 'redis', 'nginx'] ), 'so-idh': ( ['idh'] diff --git a/salt/backup/config_backup.sls b/salt/backup/config_backup.sls index a09c67b1b..a4297444b 100644 --- a/salt/backup/config_backup.sls +++ b/salt/backup/config_backup.sls @@ -32,3 +32,4 @@ so_config_backup: - daymonth: '*' - month: '*' - dayweek: '*' + diff --git a/salt/ca/files/signing_policies.conf b/salt/ca/files/signing_policies.conf index 4fc04aacc..5424d7b37 100644 --- a/salt/ca/files/signing_policies.conf +++ b/salt/ca/files/signing_policies.conf @@ -54,6 +54,20 @@ x509_signing_policies: - extendedKeyUsage: serverAuth - days_valid: 820 - copypath: /etc/pki/issued_certs/ + postgres: + - minions: '*' + - signing_private_key: /etc/pki/ca.key + - signing_cert: /etc/pki/ca.crt + - C: US + - ST: Utah + - L: Salt Lake City + - basicConstraints: "critical CA:false" + - keyUsage: "critical keyEncipherment" + - subjectKeyIdentifier: hash + - authorityKeyIdentifier: keyid,issuer:always + - extendedKeyUsage: serverAuth + - days_valid: 820 + - copypath: /etc/pki/issued_certs/ elasticfleet: - minions: '*' - signing_private_key: /etc/pki/ca.key diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common index 5ce2da241..833b9a7d8 100755 --- a/salt/common/tools/sbin/so-image-common +++ b/salt/common/tools/sbin/so-image-common @@ -31,6 +31,7 @@ container_list() { "so-hydra" "so-nginx" "so-pcaptools" + "so-postgres" "so-soc" "so-suricata" "so-telegraf" @@ -55,6 +56,7 @@ container_list() { "so-logstash" "so-nginx" "so-pcaptools" + "so-postgres" "so-redis" "so-soc" "so-strelka-backend" @@ -162,8 +164,8 @@ update_docker_containers() { # Pull down the trusted docker image run_check_net_err \ "docker pull $CONTAINER_REGISTRY/$IMAGEREPO/$image" \ - "Could not pull $image, please ensure connectivity to $CONTAINER_REGISTRY" >> "$LOG_FILE" 2>&1 - + "Could not pull $image, please ensure connectivity to $CONTAINER_REGISTRY" >> "$LOG_FILE" 2>&1 + # Get signature run_check_net_err \ "curl --retry 5 --retry-delay 60 -A '$CURLTYPE/$CURRENTVERSION/$OS/$(uname -r)' $sig_url --output $SIGNPATH/$image.sig" \ @@ -186,8 +188,27 @@ update_docker_containers() { if [ -z "$HOSTNAME" ]; then HOSTNAME=$(hostname) fi - docker tag $CONTAINER_REGISTRY/$IMAGEREPO/$image $HOSTNAME:5000/$IMAGEREPO/$image >> "$LOG_FILE" 2>&1 - docker push $HOSTNAME:5000/$IMAGEREPO/$image >> "$LOG_FILE" 2>&1 + docker tag $CONTAINER_REGISTRY/$IMAGEREPO/$image $HOSTNAME:5000/$IMAGEREPO/$image >> "$LOG_FILE" 2>&1 || { + echo "Unable to tag $image" >> "$LOG_FILE" 2>&1 + exit 1 + } + # Push to the embedded registry via a registry-to-registry copy. Avoids + # `docker push`, which on Docker 29.x with the containerd image store + # represents freshly-pulled images as an index whose layer content + # isn't reachable through the push path. The local `docker tag` above + # is preserved so so-image-pull's `:5000` existence check still works. + # Pin to the digest already gpg-verified above so we copy exactly the + # bytes we approved. + local VERIFIED_REF + VERIFIED_REF=$(echo "$DOCKERINSPECT" | jq -r ".[0].RepoDigests[] | select(. | contains(\"$CONTAINER_REGISTRY\"))" | head -n 1) + if [ -z "$VERIFIED_REF" ] || [ "$VERIFIED_REF" = "null" ]; then + echo "Unable to determine verified digest for $image" >> "$LOG_FILE" 2>&1 + exit 1 + fi + docker buildx imagetools create --tag $HOSTNAME:5000/$IMAGEREPO/$image "$VERIFIED_REF" >> "$LOG_FILE" 2>&1 || { + echo "Unable to copy $image to embedded registry" >> "$LOG_FILE" 2>&1 + exit 1 + } fi else echo "There is a problem downloading the $image image. Details: " >> "$LOG_FILE" 2>&1 diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check index 8c8bbf35c..65b1041fe 100755 --- a/salt/common/tools/sbin/so-log-check +++ b/salt/common/tools/sbin/so-log-check @@ -165,6 +165,8 @@ if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|upgrading component template" # false positive (elasticsearch index or template names contain 'error') EXCLUDED_ERRORS="$EXCLUDED_ERRORS|upgrading composable template" # false positive (elasticsearch composable template names contain 'error') EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Error while parsing document for index \[.ds-logs-kratos-so-.*object mapping for \[file\]" # false positive (mapping error occuring BEFORE kratos index has rolled over in 2.4.210) + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|No such container" # false positive (telegraf trying to run stats on an old container) + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|passwords do not match" # false positive (automated hydra test) fi if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then @@ -227,7 +229,7 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|from NIC checksum offloading" # zeek reporter.log EXCLUDED_ERRORS="$EXCLUDED_ERRORS|marked for removal" # docker container getting recycled EXCLUDED_ERRORS="$EXCLUDED_ERRORS|tcp 127.0.0.1:6791: bind: address already in use" # so-elastic-fleet agent restarting. Seen starting w/ 8.18.8 https://github.com/elastic/kibana/issues/201459 - EXCLUDED_ERRORS="$EXCLUDED_ERRORS|TransformTask\] \[logs-(tychon|aws_billing|microsoft_defender_endpoint).*user so_kibana lacks the required permissions \[logs-\1" # Known issue with 3 integrations using kibana_system role vs creating unique api creds with proper permissions. + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|TransformTask\] \[logs-(tychon|aws_billing|microsoft_defender_endpoint|armis|o365_metrics|microsoft_sentinel|snyk|cyera|island_browser).*user so_kibana lacks the required permissions \[(logs|metrics)-\1" # Known issue with integrations starting transform jobs that are explicitly not allowed to start as a system user. This error should not be seen on fresh ES 9.3.3 installs or after SO 3.1.0 with soups addition of check_transform_health_and_reauthorize() EXCLUDED_ERRORS="$EXCLUDED_ERRORS|manifest unknown" # appears in so-dockerregistry log for so-tcpreplay following docker upgrade to 29.2.1-1 fi diff --git a/salt/common/tools/sbin_jinja/so-raid-status b/salt/common/tools/sbin_jinja/so-raid-status index 3fe238c23..ca3c34608 100755 --- a/salt/common/tools/sbin_jinja/so-raid-status +++ b/salt/common/tools/sbin_jinja/so-raid-status @@ -9,7 +9,7 @@ . /usr/sbin/so-common -software_raid=("SOSMN" "SOSMN-DE02" "SOSSNNV" "SOSSNNV-DE02" "SOS10k-DE02" "SOS10KNV" "SOS10KNV-DE02" "SOS10KNV-DE02" "SOS2000-DE02" "SOS-GOFAST-LT-DE02" "SOS-GOFAST-MD-DE02" "SOS-GOFAST-HV-DE02") +software_raid=("SOSMN" "SOSMN-DE02" "SOSSNNV" "SOSSNNV-DE02" "SOS10k-DE02" "SOS10KNV" "SOS10KNV-DE02" "SOS10KNV-DE02" "SOS2000-DE02" "SOS-GOFAST-LT-DE02" "SOS-GOFAST-MD-DE02" "SOS-GOFAST-HV-DE02" "HVGUEST") hardware_raid=("SOS1000" "SOS1000F" "SOSSN7200" "SOS5000" "SOS4000") {%- if salt['grains.get']('sosmodel', '') %} @@ -87,6 +87,11 @@ check_boss_raid() { } check_software_raid() { + if [[ ! -f /proc/mdstat ]]; then + SWRAID=0 + return + fi + SWRC=$(grep "_" /proc/mdstat) if [[ -n $SWRC ]]; then # RAID is failed in some way @@ -107,7 +112,9 @@ if [[ "$is_hwraid" == "true" ]]; then fi if [[ "$is_softwareraid" == "true" ]]; then check_software_raid - check_boss_raid + if [ "$model" != "HVGUEST" ]; then + check_boss_raid + fi fi sum=$(($SWRAID + $BOSSRAID + $HWRAID)) diff --git a/salt/docker/defaults.yaml b/salt/docker/defaults.yaml index 044ec98b0..81ff07190 100644 --- a/salt/docker/defaults.yaml +++ b/salt/docker/defaults.yaml @@ -237,3 +237,11 @@ docker: extra_hosts: [] extra_env: [] ulimits: [] + 'so-postgres': + final_octet: 47 + port_bindings: + - 0.0.0.0:5432:5432 + custom_bind_mounts: [] + extra_hosts: [] + extra_env: [] + ulimits: [] diff --git a/salt/elastic-fleet-package-registry/enabled.sls b/salt/elastic-fleet-package-registry/enabled.sls index e2833f5be..a5a8b9d22 100644 --- a/salt/elastic-fleet-package-registry/enabled.sls +++ b/salt/elastic-fleet-package-registry/enabled.sls @@ -51,6 +51,16 @@ so-elastic-fleet-package-registry: - {{ ULIMIT.name }}={{ ULIMIT.soft }}:{{ ULIMIT.hard }} {% endfor %} {% endif %} + +wait_for_so-elastic-fleet-package-registry: + http.wait_for_successful_query: + - name: "http://localhost:8080/health" + - status: 200 + - wait_for: 300 + - request_interval: 15 + - require: + - docker_container: so-elastic-fleet-package-registry + delete_so-elastic-fleet-package-registry_so-status.disabled: file.uncomment: - name: /opt/so/conf/so-status/so-status.conf diff --git a/salt/elasticfleet/content-defaults.map.jinja b/salt/elasticfleet/content-defaults.map.jinja new file mode 100644 index 000000000..f4237d6d1 --- /dev/null +++ b/salt/elasticfleet/content-defaults.map.jinja @@ -0,0 +1,123 @@ +{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one + or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use + this file except in compliance with the Elastic License 2.0. #} + + +{% import_json '/opt/so/state/esfleet_content_package_components.json' as ADDON_CONTENT_PACKAGE_COMPONENTS %} +{% import_json '/opt/so/state/esfleet_component_templates.json' as INSTALLED_COMPONENT_TEMPLATES %} +{% import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %} + +{% set CORE_ESFLEET_PACKAGES = ELASTICFLEETDEFAULTS.get('elasticfleet', {}).get('packages', {}) %} +{% set ADDON_CONTENT_INTEGRATION_DEFAULTS = {} %} +{% set DEBUG_STUFF = {} %} + +{% for pkg in ADDON_CONTENT_PACKAGE_COMPONENTS %} +{% if pkg.name in CORE_ESFLEET_PACKAGES %} +{# skip core content packages #} +{% elif pkg.name not in CORE_ESFLEET_PACKAGES %} +{# generate defaults for each content package #} +{% if pkg.dataStreams is defined and pkg.dataStreams is not none and pkg.dataStreams | length > 0%} +{% for pattern in pkg.dataStreams %} +{# in ES 9.3.2 'input' type integrations no longer create default component templates and instead they wait for user input during 'integration' setup (fleet ui config) + title: generic is an artifact of that and is not in use #} +{% if pattern.title == "generic" %} +{% continue %} +{% endif %} +{% if "metrics-" in pattern.name %} +{% set integration_type = "metrics-" %} +{% elif "logs-" in pattern.name %} +{% set integration_type = "logs-" %} +{% else %} +{% set integration_type = "" %} +{% endif %} +{# on content integrations the component name is user defined at the time it is added to an agent policy #} +{% set component_name = pattern.title %} +{% set index_pattern = pattern.name %} +{# component_name_x maintains the functionality of merging local pillar changes with generated 'defaults' via SOC UI #} +{% set component_name_x = component_name.replace(".","_x_") %} +{# pillar overrides/merge expects the key names to follow the naming in elasticsearch/defaults.yaml eg. so-logs-1password_x_item_usages . The _x_ is replaced later on in elasticsearch/template.map.jinja #} +{% set integration_key = "so-" ~ integration_type ~ pkg.name + '_x_' ~ component_name_x %} +{# Default integration settings #} +{% set integration_defaults = { + "index_sorting": false, + "index_template": { + "composed_of": [integration_type ~ component_name ~ "@package", integration_type ~ component_name ~ "@custom", "so-fleet_integrations.ip_mappings-1", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"], + "data_stream": { + "allow_custom_routing": false, + "hidden": false + }, + "ignore_missing_component_templates": [integration_type ~ component_name ~ "@custom"], + "index_patterns": [index_pattern], + "priority": 501, + "template": { + "settings": { + "index": { + "lifecycle": {"name": "so-" ~ integration_type ~ component_name ~ "-logs"}, + "number_of_replicas": 0 + } + } + } + }, + "policy": { + "phases": { + "cold": { + "actions": { + "allocate":{ + "number_of_replicas": "" + }, + "set_priority": {"priority": 0} + }, + "min_age": "60d" + }, + "delete": { + "actions": { + "delete": {} + }, + "min_age": "365d" + }, + "hot": { + "actions": { + "rollover": { + "max_age": "30d", + "max_primary_shard_size": "50gb" + }, + "forcemerge":{ + "max_num_segments": "" + }, + "shrink":{ + "max_primary_shard_size": "", + "method": "COUNT", + "number_of_shards": "" + }, + "set_priority": {"priority": 100} + }, + "min_age": "0ms" + }, + "warm": { + "actions": { + "allocate": { + "number_of_replicas": "" + }, + "forcemerge": { + "max_num_segments": "" + }, + "shrink":{ + "max_primary_shard_size": "", + "method": "COUNT", + "number_of_shards": "" + }, + "set_priority": {"priority": 50} + }, + "min_age": "30d" + } + } + } + } %} + + +{% do ADDON_CONTENT_INTEGRATION_DEFAULTS.update({integration_key: integration_defaults}) %} +{% endfor %} +{% else %} +{% endif %} +{% endif %} +{% endfor %} diff --git a/salt/elasticfleet/defaults.yaml b/salt/elasticfleet/defaults.yaml index a3132d3f4..022600083 100644 --- a/salt/elasticfleet/defaults.yaml +++ b/salt/elasticfleet/defaults.yaml @@ -1,5 +1,6 @@ elasticfleet: enabled: False + patch_version: 9.3.3+build202604082258 # Elastic Agent specific patch release. enable_manager_output: True config: server: diff --git a/salt/elasticfleet/enabled.sls b/salt/elasticfleet/enabled.sls index 89ba1f80a..166cb9719 100644 --- a/salt/elasticfleet/enabled.sls +++ b/salt/elasticfleet/enabled.sls @@ -17,65 +17,19 @@ include: - logstash.ssl - elasticfleet.config - elasticfleet.sostatus +{%- if GLOBALS.role != "so-fleet" %} + - elasticfleet.manager +{%- endif %} -{% if grains.role not in ['so-fleet'] %} +{% if GLOBALS.role != "so-fleet" %} # Wait for Elasticsearch to be ready - no reason to try running Elastic Fleet server if ES is not ready wait_for_elasticsearch_elasticfleet: cmd.run: - name: so-elasticsearch-wait {% endif %} -# If enabled, automatically update Fleet Logstash Outputs -{% if ELASTICFLEETMERGED.config.server.enable_auto_configuration and grains.role not in ['so-import', 'so-eval', 'so-fleet'] %} -so-elastic-fleet-auto-configure-logstash-outputs: - cmd.run: - - name: /usr/sbin/so-elastic-fleet-outputs-update - - retry: - attempts: 4 - interval: 30 - -{# Separate from above in order to catch elasticfleet-logstash.crt changes and force update to fleet output policy #} -so-elastic-fleet-auto-configure-logstash-outputs-force: - cmd.run: - - name: /usr/sbin/so-elastic-fleet-outputs-update --certs - - retry: - attempts: 4 - interval: 30 - - onchanges: - - x509: etc_elasticfleet_logstash_crt - - x509: elasticfleet_kafka_crt -{% endif %} - -# If enabled, automatically update Fleet Server URLs & ES Connection -{% if ELASTICFLEETMERGED.config.server.enable_auto_configuration and grains.role not in ['so-fleet'] %} -so-elastic-fleet-auto-configure-server-urls: - cmd.run: - - name: /usr/sbin/so-elastic-fleet-urls-update - - retry: - attempts: 4 - interval: 30 -{% endif %} - -# Automatically update Fleet Server Elasticsearch URLs & Agent Artifact URLs -{% if grains.role not in ['so-fleet'] %} -so-elastic-fleet-auto-configure-elasticsearch-urls: - cmd.run: - - name: /usr/sbin/so-elastic-fleet-es-url-update - - retry: - attempts: 4 - interval: 30 - -so-elastic-fleet-auto-configure-artifact-urls: - cmd.run: - - name: /usr/sbin/so-elastic-fleet-artifacts-url-update - - retry: - attempts: 4 - interval: 30 - -{% endif %} - +{% if GLOBALS.role == "so-fleet" %} # Sync Elastic Agent artifacts to Fleet Node -{% if grains.role in ['so-fleet'] %} elasticagent_syncartifacts: file.recurse: - name: /nsm/elastic-fleet/artifacts/beats @@ -149,57 +103,6 @@ so-elastic-fleet: - x509: etc_elasticfleet_crt {% endif %} -{% if GLOBALS.role != "so-fleet" %} -so-elastic-fleet-package-statefile: - file.managed: - - name: /opt/so/state/elastic_fleet_packages.txt - - contents: {{ELASTICFLEETMERGED.packages}} - -so-elastic-fleet-package-upgrade: - cmd.run: - - name: /usr/sbin/so-elastic-fleet-package-upgrade - - retry: - attempts: 3 - interval: 10 - - onchanges: - - file: /opt/so/state/elastic_fleet_packages.txt - -so-elastic-fleet-integrations: - cmd.run: - - name: /usr/sbin/so-elastic-fleet-integration-policy-load - - retry: - attempts: 3 - interval: 10 - -so-elastic-agent-grid-upgrade: - cmd.run: - - name: /usr/sbin/so-elastic-agent-grid-upgrade - - retry: - attempts: 12 - interval: 5 - -so-elastic-fleet-integration-upgrade: - cmd.run: - - name: /usr/sbin/so-elastic-fleet-integration-upgrade - - retry: - attempts: 3 - interval: 10 - -{# Optional integrations script doesn't need the retries like so-elastic-fleet-integration-upgrade which loads the default integrations #} -so-elastic-fleet-addon-integrations: - cmd.run: - - name: /usr/sbin/so-elastic-fleet-optional-integrations-load - -{% if ELASTICFLEETMERGED.config.defend_filters.enable_auto_configuration %} -so-elastic-defend-manage-filters-file-watch: - cmd.run: - - name: python3 /sbin/so-elastic-defend-manage-filters.py -c /opt/so/conf/elasticsearch/curl.config -d /opt/so/conf/elastic-fleet/defend-exclusions/disabled-filters.yaml -i /nsm/securityonion-resources/event_filters/ -i /opt/so/conf/elastic-fleet/defend-exclusions/rulesets/custom-filters/ &>> /opt/so/log/elasticfleet/elastic-defend-manage-filters.log - - onchanges: - - file: elasticdefendcustom - - file: elasticdefenddisabled -{% endif %} -{% endif %} - delete_so-elastic-fleet_so-status.disabled: file.uncomment: - name: /opt/so/conf/so-status/so-status.conf diff --git a/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/import-zeek-logs.json b/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/import-zeek-logs.json index ac03f3c1d..c1fd7f147 100644 --- a/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/import-zeek-logs.json +++ b/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/import-zeek-logs.json @@ -9,16 +9,22 @@ "namespace": "so", "description": "Zeek Import logs", "policy_id": "so-grid-nodes_general", + "policy_ids": [ + "so-grid-nodes_general" + ], + "vars": {}, "inputs": { "filestream-filestream": { "enabled": true, "streams": { - "filestream.generic": { + "filestream.filestream": { "enabled": true, "vars": { "paths": [ "/nsm/import/*/zeek/logs/*.log" ], + "compression_gzip": false, + "use_logs_stream": false, "data_stream.dataset": "import", "pipeline": "", "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", @@ -34,7 +40,8 @@ "fingerprint_length": "64", "file_identity_native": true, "exclude_lines": [], - "include_lines": [] + "include_lines": [], + "delete_enabled": false } } } diff --git a/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/kratos-logs.json b/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/kratos-logs.json index 545588521..83d153439 100644 --- a/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/kratos-logs.json +++ b/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/kratos-logs.json @@ -15,19 +15,25 @@ "version": "" }, "name": "kratos-logs", + "namespace": "so", "description": "Kratos logs", "policy_id": "so-grid-nodes_general", - "namespace": "so", + "policy_ids": [ + "so-grid-nodes_general" + ], + "vars": {}, "inputs": { "filestream-filestream": { "enabled": true, "streams": { - "filestream.generic": { + "filestream.filestream": { "enabled": true, "vars": { "paths": [ "/opt/so/log/kratos/kratos.log" ], + "compression_gzip": false, + "use_logs_stream": false, "data_stream.dataset": "kratos", "pipeline": "kratos", "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", @@ -48,10 +54,10 @@ "harvester_limit": 0, "fingerprint": false, "fingerprint_offset": 0, - "fingerprint_length": "64", "file_identity_native": true, "exclude_lines": [], - "include_lines": [] + "include_lines": [], + "delete_enabled": false } } } diff --git a/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/zeek-logs.json b/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/zeek-logs.json index 4af2b2921..9797b9e75 100644 --- a/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/zeek-logs.json +++ b/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/zeek-logs.json @@ -9,16 +9,22 @@ "namespace": "so", "description": "Zeek logs", "policy_id": "so-grid-nodes_general", + "policy_ids": [ + "so-grid-nodes_general" + ], + "vars": {}, "inputs": { "filestream-filestream": { "enabled": true, "streams": { - "filestream.generic": { + "filestream.filestream": { "enabled": true, "vars": { "paths": [ "/nsm/zeek/logs/current/*.log" ], + "compression_gzip": false, + "use_logs_stream": false, "data_stream.dataset": "zeek", "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", "exclude_files": ["({%- endraw -%}{{ ELASTICFLEETMERGED.logging.zeek.excluded | join('|') }}{%- raw -%})(\\..+)?\\.log$"], @@ -30,10 +36,10 @@ "harvester_limit": 0, "fingerprint": false, "fingerprint_offset": 0, - "fingerprint_length": "64", "file_identity_native": true, "exclude_lines": [], - "include_lines": [] + "include_lines": [], + "delete_enabled": false } } } diff --git a/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json b/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json index debfc73a3..c27da26f7 100644 --- a/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json +++ b/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json @@ -5,7 +5,7 @@ "package": { "name": "endpoint", "title": "Elastic Defend", - "version": "9.0.2", + "version": "9.3.0", "requires_root": true }, "enabled": true, diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/elastic-agent-monitor.json b/salt/elasticfleet/files/integrations/grid-nodes_general/elastic-agent-monitor.json index 0be40a3d3..3eec63d26 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/elastic-agent-monitor.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/elastic-agent-monitor.json @@ -6,21 +6,23 @@ "name": "agent-monitor", "namespace": "", "description": "", + "policy_id": "so-grid-nodes_general", "policy_ids": [ "so-grid-nodes_general" ], - "output_id": null, "vars": {}, "inputs": { "filestream-filestream": { "enabled": true, "streams": { - "filestream.generic": { + "filestream.filestream": { "enabled": true, "vars": { "paths": [ "/opt/so/log/agents/agent-monitor.log" ], + "compression_gzip": false, + "use_logs_stream": false, "data_stream.dataset": "agentmonitor", "pipeline": "elasticagent.monitor", "parsers": "", @@ -34,15 +36,16 @@ "ignore_older": "72h", "clean_inactive": -1, "harvester_limit": 0, - "fingerprint": true, + "fingerprint": false, "fingerprint_offset": 0, - "fingerprint_length": 64, - "file_identity_native": false, + "file_identity_native": true, "exclude_lines": [], - "include_lines": [] + "include_lines": [], + "delete_enabled": false } } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/hydra-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/hydra-logs.json index a4f944ba5..5dcd3012d 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/hydra-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/hydra-logs.json @@ -4,19 +4,25 @@ "version": "" }, "name": "hydra-logs", + "namespace": "so", "description": "Hydra logs", "policy_id": "so-grid-nodes_general", - "namespace": "so", + "policy_ids": [ + "so-grid-nodes_general" + ], + "vars": {}, "inputs": { "filestream-filestream": { "enabled": true, "streams": { - "filestream.generic": { + "filestream.filestream": { "enabled": true, "vars": { "paths": [ "/opt/so/log/hydra/hydra.log" ], + "compression_gzip": false, + "use_logs_stream": false, "data_stream.dataset": "hydra", "pipeline": "hydra", "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", @@ -34,10 +40,10 @@ "harvester_limit": 0, "fingerprint": false, "fingerprint_offset": 0, - "fingerprint_length": "64", "file_identity_native": true, "exclude_lines": [], - "include_lines": [] + "include_lines": [], + "delete_enabled": false } } } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/idh-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/idh-logs.json index fef9c57fb..afaf77f0c 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/idh-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/idh-logs.json @@ -4,19 +4,25 @@ "version": "" }, "name": "idh-logs", + "namespace": "so", "description": "IDH integration", "policy_id": "so-grid-nodes_general", - "namespace": "so", + "policy_ids": [ + "so-grid-nodes_general" + ], + "vars": {}, "inputs": { "filestream-filestream": { "enabled": true, "streams": { - "filestream.generic": { + "filestream.filestream": { "enabled": true, "vars": { "paths": [ "/nsm/idh/opencanary.log" ], + "compression_gzip": false, + "use_logs_stream": false, "data_stream.dataset": "idh", "pipeline": "common", "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", @@ -31,10 +37,10 @@ "harvester_limit": 0, "fingerprint": false, "fingerprint_offset": 0, - "fingerprint_length": "64", "file_identity_native": true, "exclude_lines": [], - "include_lines": [] + "include_lines": [], + "delete_enabled": false } } } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json index 50ffd5dc7..32d210172 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json @@ -4,26 +4,32 @@ "version": "" }, "name": "import-evtx-logs", + "namespace": "so", "description": "Import Windows EVTX logs", "policy_id": "so-grid-nodes_general", - "namespace": "so", + "policy_ids": [ + "so-grid-nodes_general" + ], + "vars": {}, "inputs": { "filestream-filestream": { "enabled": true, "streams": { - "filestream.generic": { + "filestream.filestream": { "enabled": true, "vars": { "paths": [ "/nsm/import/*/evtx/*.json" ], + "compression_gzip": false, + "use_logs_stream": false, "data_stream.dataset": "import", "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", "exclude_files": [ "\\.gz$" ], "include_files": [], - "processors": "- dissect:\n tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"\n- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n- drop_fields:\n fields: [\"host\"]\n ignore_missing: true\n- add_fields:\n target: data_stream\n fields:\n type: logs\n dataset: system.security\n- add_fields:\n target: event\n fields:\n dataset: system.security\n module: system\n imported: true\n- add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.security-2.6.1\n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.sysmon_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.sysmon_operational\n module: windows\n imported: true\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.sysmon_operational-3.1.2\n- if:\n equals:\n winlog.channel: 'Application'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.application\n - add_fields:\n target: event\n fields:\n dataset: system.application\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.application-2.6.1\n- if:\n equals:\n winlog.channel: 'System'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.system\n - add_fields:\n target: event\n fields:\n dataset: system.system\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.system-2.6.1\n \n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.powershell_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.powershell_operational\n module: windows\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.powershell_operational-3.1.2\n- add_fields:\n target: data_stream\n fields:\n dataset: import", + "processors": "- dissect:\n tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"\n- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n- drop_fields:\n fields: [\"host\"]\n ignore_missing: true\n- add_fields:\n target: data_stream\n fields:\n type: logs\n dataset: system.security\n- add_fields:\n target: event\n fields:\n dataset: system.security\n module: system\n imported: true\n- add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.security-2.15.0\n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.sysmon_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.sysmon_operational\n module: windows\n imported: true\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.sysmon_operational-3.8.0\n- if:\n equals:\n winlog.channel: 'Application'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.application\n - add_fields:\n target: event\n fields:\n dataset: system.application\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.application-2.15.0\n- if:\n equals:\n winlog.channel: 'System'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.system\n - add_fields:\n target: event\n fields:\n dataset: system.system\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.system-2.15.0\n \n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.powershell_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.powershell_operational\n module: windows\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.powershell_operational-3.8.0\n- add_fields:\n target: data_stream\n fields:\n dataset: import", "tags": [ "import" ], @@ -33,10 +39,10 @@ "harvester_limit": 0, "fingerprint": false, "fingerprint_offset": 0, - "fingerprint_length": "64", "file_identity_native": true, "exclude_lines": [], - "include_lines": [] + "include_lines": [], + "delete_enabled": false } } } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/import-suricata-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/import-suricata-logs.json index b8f3b0b29..3148b38e8 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/import-suricata-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/import-suricata-logs.json @@ -4,19 +4,25 @@ "version": "" }, "name": "import-suricata-logs", + "namespace": "so", "description": "Import Suricata logs", "policy_id": "so-grid-nodes_general", - "namespace": "so", + "policy_ids": [ + "so-grid-nodes_general" + ], + "vars": {}, "inputs": { "filestream-filestream": { "enabled": true, "streams": { - "filestream.generic": { + "filestream.filestream": { "enabled": true, "vars": { "paths": [ "/nsm/import/*/suricata/eve*.json" ], + "compression_gzip": false, + "use_logs_stream": false, "data_stream.dataset": "import", "pipeline": "suricata.common", "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", @@ -32,10 +38,10 @@ "harvester_limit": 0, "fingerprint": false, "fingerprint_offset": 0, - "fingerprint_length": "64", "file_identity_native": true, "exclude_lines": [], - "include_lines": [] + "include_lines": [], + "delete_enabled": false } } } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/rita-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/rita-logs.json index 70259c3cf..f807c3b70 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/rita-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/rita-logs.json @@ -4,14 +4,18 @@ "version": "" }, "name": "rita-logs", + "namespace": "so", "description": "RITA Logs", "policy_id": "so-grid-nodes_general", - "namespace": "so", + "policy_ids": [ + "so-grid-nodes_general" + ], + "vars": {}, "inputs": { "filestream-filestream": { "enabled": true, "streams": { - "filestream.generic": { + "filestream.filestream": { "enabled": true, "vars": { "paths": [ @@ -19,6 +23,8 @@ "/nsm/rita/exploded-dns.csv", "/nsm/rita/long-connections.csv" ], + "compression_gzip": false, + "use_logs_stream": false, "data_stream.dataset": "rita", "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", "exclude_files": [ @@ -33,10 +39,10 @@ "harvester_limit": 0, "fingerprint": false, "fingerprint_offset": 0, - "fingerprint_length": "64", "file_identity_native": true, "exclude_lines": [], - "include_lines": [] + "include_lines": [], + "delete_enabled": false } } } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/so-ip-mappings.json b/salt/elasticfleet/files/integrations/grid-nodes_general/so-ip-mappings.json index a14e63559..24ed188f2 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/so-ip-mappings.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/so-ip-mappings.json @@ -4,19 +4,25 @@ "version": "" }, "name": "so-ip-mappings", + "namespace": "so", "description": "IP Description mappings", "policy_id": "so-grid-nodes_general", - "namespace": "so", + "policy_ids": [ + "so-grid-nodes_general" + ], + "vars": {}, "inputs": { "filestream-filestream": { "enabled": true, "streams": { - "filestream.generic": { + "filestream.filestream": { "enabled": true, "vars": { "paths": [ "/nsm/custom-mappings/ip-descriptions.csv" ], + "compression_gzip": false, + "use_logs_stream": false, "data_stream.dataset": "hostnamemappings", "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", "exclude_files": [ @@ -32,10 +38,10 @@ "harvester_limit": 0, "fingerprint": false, "fingerprint_offset": 0, - "fingerprint_length": "64", "file_identity_native": true, "exclude_lines": [], - "include_lines": [] + "include_lines": [], + "delete_enabled": false } } } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-auth-sync-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-auth-sync-logs.json index f4fd38e9d..c04b738d3 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-auth-sync-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-auth-sync-logs.json @@ -4,19 +4,25 @@ "version": "" }, "name": "soc-auth-sync-logs", + "namespace": "so", "description": "Security Onion - Elastic Auth Sync - Logs", "policy_id": "so-grid-nodes_general", - "namespace": "so", + "policy_ids": [ + "so-grid-nodes_general" + ], + "vars": {}, "inputs": { "filestream-filestream": { "enabled": true, "streams": { - "filestream.generic": { + "filestream.filestream": { "enabled": true, "vars": { "paths": [ "/opt/so/log/soc/sync.log" ], + "compression_gzip": false, + "use_logs_stream": false, "data_stream.dataset": "soc", "pipeline": "common", "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", @@ -31,10 +37,10 @@ "harvester_limit": 0, "fingerprint": false, "fingerprint_offset": 0, - "fingerprint_length": "64", "file_identity_native": true, "exclude_lines": [], - "include_lines": [] + "include_lines": [], + "delete_enabled": false } } } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-detections-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-detections-logs.json index f1bdbc922..9d7812e42 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-detections-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-detections-logs.json @@ -4,20 +4,26 @@ "version": "" }, "name": "soc-detections-logs", + "namespace": "so", "description": "Security Onion Console - Detections Logs", "policy_id": "so-grid-nodes_general", - "namespace": "so", + "policy_ids": [ + "so-grid-nodes_general" + ], + "vars": {}, "inputs": { "filestream-filestream": { "enabled": true, "streams": { - "filestream.generic": { + "filestream.filestream": { "enabled": true, "vars": { "paths": [ "/opt/so/log/soc/detections_runtime-status_sigma.log", "/opt/so/log/soc/detections_runtime-status_yara.log" ], + "compression_gzip": false, + "use_logs_stream": false, "data_stream.dataset": "soc", "pipeline": "common", "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", @@ -35,10 +41,10 @@ "harvester_limit": 0, "fingerprint": false, "fingerprint_offset": 0, - "fingerprint_length": "64", "file_identity_native": true, "exclude_lines": [], - "include_lines": [] + "include_lines": [], + "delete_enabled": false } } } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-salt-relay-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-salt-relay-logs.json index cb08d5b12..d1fa8b630 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-salt-relay-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-salt-relay-logs.json @@ -4,19 +4,25 @@ "version": "" }, "name": "soc-salt-relay-logs", + "namespace": "so", "description": "Security Onion - Salt Relay - Logs", "policy_id": "so-grid-nodes_general", - "namespace": "so", + "policy_ids": [ + "so-grid-nodes_general" + ], + "vars": {}, "inputs": { "filestream-filestream": { "enabled": true, "streams": { - "filestream.generic": { + "filestream.filestream": { "enabled": true, "vars": { "paths": [ "/opt/so/log/soc/salt-relay.log" ], + "compression_gzip": false, + "use_logs_stream": false, "data_stream.dataset": "soc", "pipeline": "common", "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", @@ -33,10 +39,10 @@ "harvester_limit": 0, "fingerprint": false, "fingerprint_offset": 0, - "fingerprint_length": "64", "file_identity_native": true, "exclude_lines": [], - "include_lines": [] + "include_lines": [], + "delete_enabled": false } } } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-sensoroni-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-sensoroni-logs.json index 11e686c3d..467544c9d 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-sensoroni-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-sensoroni-logs.json @@ -4,19 +4,25 @@ "version": "" }, "name": "soc-sensoroni-logs", + "namespace": "so", "description": "Security Onion - Sensoroni - Logs", "policy_id": "so-grid-nodes_general", - "namespace": "so", + "policy_ids": [ + "so-grid-nodes_general" + ], + "vars": {}, "inputs": { "filestream-filestream": { "enabled": true, "streams": { - "filestream.generic": { + "filestream.filestream": { "enabled": true, "vars": { "paths": [ "/opt/so/log/sensoroni/sensoroni.log" ], + "compression_gzip": false, + "use_logs_stream": false, "data_stream.dataset": "soc", "pipeline": "common", "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", @@ -31,10 +37,10 @@ "harvester_limit": 0, "fingerprint": false, "fingerprint_offset": 0, - "fingerprint_length": "64", "file_identity_native": true, "exclude_lines": [], - "include_lines": [] + "include_lines": [], + "delete_enabled": false } } } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-server-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-server-logs.json index decb6b22a..37eb02ab1 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-server-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-server-logs.json @@ -4,19 +4,25 @@ "version": "" }, "name": "soc-server-logs", + "namespace": "so", "description": "Security Onion Console Logs", "policy_id": "so-grid-nodes_general", - "namespace": "so", + "policy_ids": [ + "so-grid-nodes_general" + ], + "vars": {}, "inputs": { "filestream-filestream": { "enabled": true, "streams": { - "filestream.generic": { + "filestream.filestream": { "enabled": true, "vars": { "paths": [ "/opt/so/log/soc/sensoroni-server.log" ], + "compression_gzip": false, + "use_logs_stream": false, "data_stream.dataset": "soc", "pipeline": "common", "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", @@ -33,10 +39,10 @@ "harvester_limit": 0, "fingerprint": false, "fingerprint_offset": 0, - "fingerprint_length": "64", "file_identity_native": true, "exclude_lines": [], - "include_lines": [] + "include_lines": [], + "delete_enabled": false } } } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/strelka-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/strelka-logs.json index 1f0203a91..3091baf44 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/strelka-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/strelka-logs.json @@ -4,19 +4,25 @@ "version": "" }, "name": "strelka-logs", + "namespace": "so", "description": "Strelka Logs", "policy_id": "so-grid-nodes_general", - "namespace": "so", + "policy_ids": [ + "so-grid-nodes_general" + ], + "vars": {}, "inputs": { "filestream-filestream": { "enabled": true, "streams": { - "filestream.generic": { + "filestream.filestream": { "enabled": true, "vars": { "paths": [ "/nsm/strelka/log/strelka.log" ], + "compression_gzip": false, + "use_logs_stream": false, "data_stream.dataset": "strelka", "pipeline": "strelka.file", "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", @@ -31,10 +37,10 @@ "harvester_limit": 0, "fingerprint": false, "fingerprint_offset": 0, - "fingerprint_length": "64", "file_identity_native": true, "exclude_lines": [], - "include_lines": [] + "include_lines": [], + "delete_enabled": false } } } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/suricata-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/suricata-logs.json index 26dae5225..bb5cfd2c3 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/suricata-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/suricata-logs.json @@ -4,19 +4,25 @@ "version": "" }, "name": "suricata-logs", + "namespace": "so", "description": "Suricata integration", "policy_id": "so-grid-nodes_general", - "namespace": "so", + "policy_ids": [ + "so-grid-nodes_general" + ], + "vars": {}, "inputs": { "filestream-filestream": { "enabled": true, "streams": { - "filestream.generic": { + "filestream.filestream": { "enabled": true, "vars": { "paths": [ "/nsm/suricata/eve*.json" ], + "compression_gzip": false, + "use_logs_stream": false, "data_stream.dataset": "suricata", "pipeline": "suricata.common", "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", @@ -31,10 +37,10 @@ "harvester_limit": 0, "fingerprint": false, "fingerprint_offset": 0, - "fingerprint_length": "64", "file_identity_native": true, "exclude_lines": [], - "include_lines": [] + "include_lines": [], + "delete_enabled": false } } } diff --git a/salt/elasticfleet/input-defaults.map.jinja b/salt/elasticfleet/input-defaults.map.jinja new file mode 100644 index 000000000..a02844330 --- /dev/null +++ b/salt/elasticfleet/input-defaults.map.jinja @@ -0,0 +1,123 @@ +{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one + or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use + this file except in compliance with the Elastic License 2.0. #} + + +{% import_json '/opt/so/state/esfleet_input_package_components.json' as ADDON_INPUT_PACKAGE_COMPONENTS %} +{% import_json '/opt/so/state/esfleet_component_templates.json' as INSTALLED_COMPONENT_TEMPLATES %} +{% import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %} + +{% set CORE_ESFLEET_PACKAGES = ELASTICFLEETDEFAULTS.get('elasticfleet', {}).get('packages', {}) %} +{% set ADDON_INPUT_INTEGRATION_DEFAULTS = {} %} +{% set DEBUG_STUFF = {} %} + +{% for pkg in ADDON_INPUT_PACKAGE_COMPONENTS %} +{% if pkg.name in CORE_ESFLEET_PACKAGES %} +{# skip core input packages #} +{% elif pkg.name not in CORE_ESFLEET_PACKAGES %} +{# generate defaults for each input package #} +{% if pkg.dataStreams is defined and pkg.dataStreams is not none and pkg.dataStreams | length > 0 %} +{% for pattern in pkg.dataStreams %} +{# in ES 9.3.2 'input' type integrations no longer create default component templates and instead they wait for user input during 'integration' setup (fleet ui config) + title: generic is an artifact of that and is not in use #} +{% if pattern.title == "generic" %} +{% continue %} +{% endif %} +{% if "metrics-" in pattern.name %} +{% set integration_type = "metrics-" %} +{% elif "logs-" in pattern.name %} +{% set integration_type = "logs-" %} +{% else %} +{% set integration_type = "" %} +{% endif %} +{# on input integrations the component name is user defined at the time it is added to an agent policy #} +{% set component_name = pattern.title %} +{% set index_pattern = pattern.name %} +{# component_name_x maintains the functionality of merging local pillar changes with generated 'defaults' via SOC UI #} +{% set component_name_x = component_name.replace(".","_x_") %} +{# pillar overrides/merge expects the key names to follow the naming in elasticsearch/defaults.yaml eg. so-logs-1password_x_item_usages . The _x_ is replaced later on in elasticsearch/template.map.jinja #} +{% set integration_key = "so-" ~ integration_type ~ pkg.name + '_x_' ~ component_name_x %} +{# Default integration settings #} +{% set integration_defaults = { + "index_sorting": false, + "index_template": { + "composed_of": [integration_type ~ component_name ~ "@package", integration_type ~ component_name ~ "@custom", "so-fleet_integrations.ip_mappings-1", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"], + "data_stream": { + "allow_custom_routing": false, + "hidden": false + }, + "ignore_missing_component_templates": [integration_type ~ component_name ~ "@custom"], + "index_patterns": [index_pattern], + "priority": 501, + "template": { + "settings": { + "index": { + "lifecycle": {"name": "so-" ~ integration_type ~ component_name ~ "-logs"}, + "number_of_replicas": 0 + } + } + } + }, + "policy": { + "phases": { + "cold": { + "actions": { + "allocate":{ + "number_of_replicas": "" + }, + "set_priority": {"priority": 0} + }, + "min_age": "60d" + }, + "delete": { + "actions": { + "delete": {} + }, + "min_age": "365d" + }, + "hot": { + "actions": { + "rollover": { + "max_age": "30d", + "max_primary_shard_size": "50gb" + }, + "forcemerge":{ + "max_num_segments": "" + }, + "shrink":{ + "max_primary_shard_size": "", + "method": "COUNT", + "number_of_shards": "" + }, + "set_priority": {"priority": 100} + }, + "min_age": "0ms" + }, + "warm": { + "actions": { + "allocate": { + "number_of_replicas": "" + }, + "forcemerge": { + "max_num_segments": "" + }, + "shrink":{ + "max_primary_shard_size": "", + "method": "COUNT", + "number_of_shards": "" + }, + "set_priority": {"priority": 50} + }, + "min_age": "30d" + } + } + } + } %} + + +{% do ADDON_INPUT_INTEGRATION_DEFAULTS.update({integration_key: integration_defaults}) %} +{% do DEBUG_STUFF.update({integration_key: "Generating defaults for "+ pkg.name })%} +{% endfor %} +{% endif %} +{% endif %} +{% endfor %} diff --git a/salt/elasticfleet/integration-defaults.map.jinja b/salt/elasticfleet/integration-defaults.map.jinja index f85a95ec9..eeb85123a 100644 --- a/salt/elasticfleet/integration-defaults.map.jinja +++ b/salt/elasticfleet/integration-defaults.map.jinja @@ -59,8 +59,8 @@ {# skip core integrations #} {% elif pkg.name not in CORE_ESFLEET_PACKAGES %} {# generate defaults for each integration #} -{% if pkg.es_index_patterns is defined and pkg.es_index_patterns is not none %} -{% for pattern in pkg.es_index_patterns %} +{% if pkg.dataStreams is defined and pkg.dataStreams is not none and pkg.dataStreams | length > 0 %} +{% for pattern in pkg.dataStreams %} {% if "metrics-" in pattern.name %} {% set integration_type = "metrics-" %} {% elif "logs-" in pattern.name %} @@ -75,44 +75,27 @@ {% if component_name in WEIRD_INTEGRATIONS %} {% set component_name = WEIRD_INTEGRATIONS[component_name] %} {% endif %} - -{# create duplicate of component_name, so we can split generics from @custom component templates in the index template below and overwrite the default @package when needed - eg. having to replace unifiedlogs.generic@package with filestream.generic@package, but keep the ability to customize unifiedlogs.generic@custom and its ILM policy #} -{% set custom_component_name = component_name %} - -{# duplicate integration_type to assist with sometimes needing to overwrite component templates with 'logs-filestream.generic@package' (there is no metrics-filestream.generic@package) #} -{% set generic_integration_type = integration_type %} - {# component_name_x maintains the functionality of merging local pillar changes with generated 'defaults' via SOC UI #} {% set component_name_x = component_name.replace(".","_x_") %} {# pillar overrides/merge expects the key names to follow the naming in elasticsearch/defaults.yaml eg. so-logs-1password_x_item_usages . The _x_ is replaced later on in elasticsearch/template.map.jinja #} {% set integration_key = "so-" ~ integration_type ~ component_name_x %} -{# if its a .generic template make sure that a .generic@package for the integration exists. Else default to logs-filestream.generic@package #} -{% if ".generic" in component_name and integration_type ~ component_name ~ "@package" not in INSTALLED_COMPONENT_TEMPLATES %} -{# these generic templates by default are directed to index_pattern of 'logs-generic-*', overwrite that here to point to eg gcp_pubsub.generic-* #} -{% set index_pattern = integration_type ~ component_name ~ "-*" %} -{# includes use of .generic component template, but it doesn't exist in installed component templates. Redirect it to filestream.generic@package #} -{% set component_name = "filestream.generic" %} -{% set generic_integration_type = "logs-" %} -{% endif %} - {# Default integration settings #} {% set integration_defaults = { "index_sorting": false, "index_template": { - "composed_of": [generic_integration_type ~ component_name ~ "@package", integration_type ~ custom_component_name ~ "@custom", "so-fleet_integrations.ip_mappings-1", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"], + "composed_of": [integration_type ~ component_name ~ "@package", integration_type ~ component_name ~ "@custom", "so-fleet_integrations.ip_mappings-1", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"], "data_stream": { "allow_custom_routing": false, "hidden": false }, - "ignore_missing_component_templates": [integration_type ~ custom_component_name ~ "@custom"], + "ignore_missing_component_templates": [integration_type ~ component_name ~ "@custom"], "index_patterns": [index_pattern], "priority": 501, "template": { "settings": { "index": { - "lifecycle": {"name": "so-" ~ integration_type ~ custom_component_name ~ "-logs"}, + "lifecycle": {"name": "so-" ~ integration_type ~ component_name ~ "-logs"}, "number_of_replicas": 0 } } diff --git a/salt/elasticfleet/manager.sls b/salt/elasticfleet/manager.sls new file mode 100644 index 000000000..1728f2010 --- /dev/null +++ b/salt/elasticfleet/manager.sls @@ -0,0 +1,101 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{% from 'allowed_states.map.jinja' import allowed_states %} +{% if sls in allowed_states %} +{% from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %} + +include: + - elasticfleet.config + +# If enabled, automatically update Fleet Logstash Outputs +{% if ELASTICFLEETMERGED.config.server.enable_auto_configuration and grains.role not in ['so-import', 'so-eval'] %} +so-elastic-fleet-auto-configure-logstash-outputs: + cmd.run: + - name: /usr/sbin/so-elastic-fleet-outputs-update + - retry: + attempts: 4 + interval: 30 +{% endif %} + +# If enabled, automatically update Fleet Server URLs & ES Connection +so-elastic-fleet-auto-configure-server-urls: + cmd.run: + - name: /usr/sbin/so-elastic-fleet-urls-update + - retry: + attempts: 4 + interval: 30 + +# Automatically update Fleet Server Elasticsearch URLs & Agent Artifact URLs +so-elastic-fleet-auto-configure-elasticsearch-urls: + cmd.run: + - name: /usr/sbin/so-elastic-fleet-es-url-update + - retry: + attempts: 4 + interval: 30 + +so-elastic-fleet-auto-configure-artifact-urls: + cmd.run: + - name: /usr/sbin/so-elastic-fleet-artifacts-url-update + - retry: + attempts: 4 + interval: 30 + +so-elastic-fleet-package-statefile: + file.managed: + - name: /opt/so/state/elastic_fleet_packages.txt + - contents: {{ELASTICFLEETMERGED.packages}} + +so-elastic-fleet-package-upgrade: + cmd.run: + - name: /usr/sbin/so-elastic-fleet-package-upgrade + - retry: + attempts: 3 + interval: 10 + - onchanges: + - file: /opt/so/state/elastic_fleet_packages.txt + +so-elastic-fleet-integrations: + cmd.run: + - name: /usr/sbin/so-elastic-fleet-integration-policy-load + - retry: + attempts: 3 + interval: 10 + +so-elastic-agent-grid-upgrade: + cmd.run: + - name: /usr/sbin/so-elastic-agent-grid-upgrade + - retry: + attempts: 12 + interval: 5 + +so-elastic-fleet-integration-upgrade: + cmd.run: + - name: /usr/sbin/so-elastic-fleet-integration-upgrade + - retry: + attempts: 3 + interval: 10 + +{# Optional integrations script doesn't need the retries like so-elastic-fleet-integration-upgrade which loads the default integrations #} +so-elastic-fleet-addon-integrations: + cmd.run: + - name: /usr/sbin/so-elastic-fleet-optional-integrations-load + +{% if ELASTICFLEETMERGED.config.defend_filters.enable_auto_configuration %} +so-elastic-defend-manage-filters-file-watch: + cmd.run: + - name: python3 /sbin/so-elastic-defend-manage-filters.py -c /opt/so/conf/elasticsearch/curl.config -d /opt/so/conf/elastic-fleet/defend-exclusions/disabled-filters.yaml -i /nsm/securityonion-resources/event_filters/ -i /opt/so/conf/elastic-fleet/defend-exclusions/rulesets/custom-filters/ &>> /opt/so/log/elasticfleet/elastic-defend-manage-filters.log + - onchanges: + - file: elasticdefendcustom + - file: elasticdefenddisabled +{% endif %} + +{% else %} + +{{sls}}_state_not_allowed: + test.fail_without_changes: + - name: {{sls}}_state_not_allowed + +{% endif %} diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common index 1a597b1db..91fa787f2 100644 --- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common +++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common @@ -135,9 +135,33 @@ elastic_fleet_bulk_package_install() { fi } -elastic_fleet_installed_packages() { - if ! fleet_api "epm/packages/installed?perPage=500"; then +elastic_fleet_get_package_list_by_type() { + if ! output=$(fleet_api "epm/packages"); then return 1 + else + is_integration=$(jq '[.items[] | select(.type=="integration") | .name ]' <<< "$output") + is_input=$(jq '[.items[] | select(.type=="input") | .name ]' <<< "$output") + is_content=$(jq '[.items[] | select(.type=="content") | .name ]' <<< "$output") + jq -n --argjson is_integration "${is_integration:-[]}" \ + --argjson is_input "${is_input:-[]}" \ + --argjson is_content "${is_content:-[]}" \ + '{"integration": $is_integration,"input": $is_input, "content": $is_content}' + fi +} +elastic_fleet_installed_packages_components() { + package_type=${1,,} + if [[ "$package_type" != "integration" && "$package_type" != "input" && "$package_type" != "content" ]]; then + echo "Error: Invalid package type ${package_type}. Valid types are 'integration', 'input', or 'content'." + return 1 + fi + + packages_by_type=$(elastic_fleet_get_package_list_by_type) + packages=$(jq --arg package_type "$package_type" '.[$package_type]' <<< "$packages_by_type") + + if ! output=$(fleet_api "epm/packages/installed?perPage=500"); then + return 1 + else + jq -c --argjson packages "$packages" '[.items[] | select(.name | IN($packages[])) | {name: .name, dataStreams: .dataStreams}]' <<< "$output" fi } @@ -216,7 +240,7 @@ elastic_fleet_policy_create() { --arg DESC "$DESC" \ --arg TIMEOUT $TIMEOUT \ --arg FLEETSERVER "$FLEETSERVER" \ - '{"name": $NAME,"id":$NAME,"description":$DESC,"namespace":"default","monitoring_enabled":["logs"],"inactivity_timeout":$TIMEOUT,"has_fleet_server":$FLEETSERVER}' + '{"name": $NAME,"id":$NAME,"description":$DESC,"namespace":"default","monitoring_enabled":["logs"],"inactivity_timeout":$TIMEOUT,"has_fleet_server":$FLEETSERVER,"advanced_settings":{"agent_logging_level": "warning"}}' ) # Create Fleet Policy if ! fleet_api "agent_policies" -XPOST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"; then diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-grid-upgrade b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-grid-upgrade index 0729531d3..01c176adc 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-grid-upgrade +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-grid-upgrade @@ -5,7 +5,13 @@ # this file except in compliance with the Elastic License 2.0. . /usr/sbin/so-common +. /usr/sbin/so-elastic-fleet-common {%- import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS %} +{%- import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %} +{# Optionally override Elasticsearch version for Elastic Agent patch releases #} +{%- if ELASTICFLEETDEFAULTS.elasticfleet.patch_version is defined %} +{%- do ELASTICSEARCHDEFAULTS.elasticsearch.update({'version': ELASTICFLEETDEFAULTS.elasticfleet.patch_version}) %} +{%- endif %} # Only run on Managers if ! is_manager_node; then @@ -14,13 +20,10 @@ if ! is_manager_node; then fi # Get current list of Grid Node Agents that need to be upgraded -RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/agents?perPage=20&page=1&kuery=NOT%20agent.version%3A%20{{ELASTICSEARCHDEFAULTS.elasticsearch.version}}%20AND%20policy_id%3A%20so-grid-nodes_%2A&showInactive=false&getStatusSummary=true" --retry 3 --retry-delay 30 --fail 2>/dev/null) +if ! RAW_JSON=$(fleet_api "agents?perPage=20&page=1&kuery=NOT%20agent.version%3A%20{{ELASTICSEARCHDEFAULTS.elasticsearch.version | urlencode }}%20AND%20policy_id%3A%20so-grid-nodes_%2A&showInactive=false&getStatusSummary=true" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'); then -# Check to make sure that the server responded with good data - else, bail from script -CHECKSUM=$(jq -r '.page' <<< "$RAW_JSON") -if [ "$CHECKSUM" -ne 1 ]; then - printf "Failed to query for current Grid Agents...\n" - exit 1 + printf "Failed to query for current Grid Agents...\n" + exit 1 fi # Generate list of Node Agents that need updates @@ -31,10 +34,12 @@ if [ "$OUTDATED_LIST" != '[]' ]; then printf "Initiating upgrades for $AGENTNUMBERS Agents to Elastic {{ELASTICSEARCHDEFAULTS.elasticsearch.version}}...\n\n" # Generate updated JSON payload - JSON_STRING=$(jq -n --arg ELASTICVERSION {{ELASTICSEARCHDEFAULTS.elasticsearch.version}} --arg UPDATELIST $OUTDATED_LIST '{"version": $ELASTICVERSION,"agents": $UPDATELIST }') + JSON_STRING=$(jq -n --arg ELASTICVERSION "{{ELASTICSEARCHDEFAULTS.elasticsearch.version}}" --argjson UPDATELIST "$OUTDATED_LIST" '{"version": $ELASTICVERSION,"agents": $UPDATELIST }') # Update Node Agents - curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "http://localhost:5601/api/fleet/agents/bulk_upgrade" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" + if ! fleet_api "agents/bulk_upgrade" -XPOST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"; then + printf "Failed to initiate Agent upgrades...\n" + fi else printf "No Agents need updates... Exiting\n\n" exit 0 diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load index 8c0f627ef..ab38b7065 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load @@ -18,7 +18,9 @@ INSTALLED_PACKAGE_LIST=/tmp/esfleet_installed_packages.json BULK_INSTALL_PACKAGE_LIST=/tmp/esfleet_bulk_install.json BULK_INSTALL_PACKAGE_TMP=/tmp/esfleet_bulk_install_tmp.json BULK_INSTALL_OUTPUT=/opt/so/state/esfleet_bulk_install_results.json -PACKAGE_COMPONENTS=/opt/so/state/esfleet_package_components.json +INTEGRATION_PACKAGE_COMPONENTS=/opt/so/state/esfleet_package_components.json +INPUT_PACKAGE_COMPONENTS=/opt/so/state/esfleet_input_package_components.json +CONTENT_PACKAGE_COMPONENTS=/opt/so/state/esfleet_content_package_components.json COMPONENT_TEMPLATES=/opt/so/state/esfleet_component_templates.json PENDING_UPDATE=false @@ -179,10 +181,13 @@ if [[ -f $STATE_FILE_SUCCESS ]]; then else echo "Elastic integrations don't appear to need installation/updating..." fi - # Write out file for generating index/component/ilm templates - if latest_installed_package_list=$(elastic_fleet_installed_packages); then - echo $latest_installed_package_list | jq '[.items[] | {name: .name, es_index_patterns: .dataStreams}]' > $PACKAGE_COMPONENTS - fi + # Write out file for generating index/component/ilm templates, keeping each package type separate + for package_type in "INTEGRATION" "INPUT" "CONTENT"; do + if latest_installed_package_list=$(elastic_fleet_installed_packages_components "$package_type"); then + outfile="${package_type}_PACKAGE_COMPONENTS" + echo $latest_installed_package_list > "${!outfile}" + fi + done if retry 3 1 "so-elasticsearch-query / --fail --output /dev/null"; then # Refresh installed component template list latest_component_templates_list=$(so-elasticsearch-query _component_template | jq '.component_templates[] | .name' | jq -s '.') diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update index f045bf753..8630799d8 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update @@ -235,6 +235,16 @@ function update_kafka_outputs() { {% endif %} +# Compare the current Elastic Fleet certificate against what is on disk +POLICY_CERT_SHA=$(jq -r '.item.ssl.certificate' <<< $RAW_JSON | openssl x509 -noout -sha256 -fingerprint) +DISK_CERT_SHA=$(openssl x509 -in /etc/pki/elasticfleet-logstash.crt -noout -sha256 -fingerprint) + +if [[ "$POLICY_CERT_SHA" != "$DISK_CERT_SHA" ]]; then + printf "Certificate on disk doesn't match certificate in policy - forcing update\n" + UPDATE_CERTS=true + FORCE_UPDATE=true +fi + # Sort & hash the new list of Logstash Outputs NEW_LIST_JSON=$(jq --compact-output --null-input '$ARGS.positional' --args -- "${NEW_LIST[@]}") NEW_HASH=$(sha256sum <<< "$NEW_LIST_JSON" | awk '{print $1}') diff --git a/salt/elasticsearch/cluster.sls b/salt/elasticsearch/cluster.sls new file mode 100644 index 000000000..e25aed36a --- /dev/null +++ b/salt/elasticsearch/cluster.sls @@ -0,0 +1,164 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{% from 'allowed_states.map.jinja' import allowed_states %} +{% if sls in allowed_states %} +{% from 'vars/globals.map.jinja' import GLOBALS %} +{% from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %} +{% from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS, SO_MANAGED_INDICES %} +{% if GLOBALS.role != 'so-heavynode' %} +{% from 'elasticsearch/template.map.jinja' import ALL_ADDON_SETTINGS %} +{% endif %} + +escomponenttemplates: + file.recurse: + - name: /opt/so/conf/elasticsearch/templates/component + - source: salt://elasticsearch/templates/component + - user: 930 + - group: 939 + - clean: True + - onchanges_in: + - file: so-elasticsearch-templates-reload + - show_changes: False + +# Clean up legacy and non-SO managed templates from the elasticsearch/templates/index/ directory +so_index_template_dir: + file.directory: + - name: /opt/so/conf/elasticsearch/templates/index + - clean: True + {%- if SO_MANAGED_INDICES %} + - require: + {%- for index in SO_MANAGED_INDICES %} + - file: so_index_template_{{index}} + {%- endfor %} + {%- endif %} + +# Auto-generate index templates for SO managed indices (directly defined in elasticsearch/defaults.yaml) +# These index templates are for the core SO datasets and are always required +{% for index, settings in ES_INDEX_SETTINGS.items() %} +{% if settings.index_template is defined %} +so_index_template_{{index}}: + file.managed: + - name: /opt/so/conf/elasticsearch/templates/index/{{ index }}-template.json + - source: salt://elasticsearch/base-template.json.jinja + - defaults: + TEMPLATE_CONFIG: {{ settings.index_template }} + - template: jinja + - onchanges_in: + - file: so-elasticsearch-templates-reload +{% endif %} +{% endfor %} + +{% if GLOBALS.role != "so-heavynode" %} +# Auto-generate optional index templates for integration | input | content packages +# These index templates are not used by default (until user adds package to an agent policy). +# Pre-configured with standard defaults, and incorporated into SOC configuration for user customization. +{% for index,settings in ALL_ADDON_SETTINGS.items() %} +{% if settings.index_template is defined %} +addon_index_template_{{index}}: + file.managed: + - name: /opt/so/conf/elasticsearch/templates/addon-index/{{ index }}-template.json + - source: salt://elasticsearch/base-template.json.jinja + - defaults: + TEMPLATE_CONFIG: {{ settings.index_template }} + - template: jinja + - show_changes: False + - onchanges_in: + - file: addon-elasticsearch-templates-reload +{% endif %} +{% endfor %} +{% endif %} + +{% if GLOBALS.role in GLOBALS.manager_roles %} +so-es-cluster-settings: + cmd.run: + - name: /usr/sbin/so-elasticsearch-cluster-settings + - cwd: /opt/so + - template: jinja + - require: + - docker_container: so-elasticsearch + - file: elasticsearch_sbin_jinja + - http: wait_for_so-elasticsearch +{% endif %} + +# heavynodes will only load ILM policies for SO managed indices. (Indicies defined in elasticsearch/defaults.yaml) +so-elasticsearch-ilm-policy-load: + cmd.run: + - name: /usr/sbin/so-elasticsearch-ilm-policy-load + - cwd: /opt/so + - require: + - docker_container: so-elasticsearch + - file: so-elasticsearch-ilm-policy-load-script + - onchanges: + - file: so-elasticsearch-ilm-policy-load-script + +so-elasticsearch-templates-reload: + file.absent: + - name: /opt/so/state/estemplates.txt + +addon-elasticsearch-templates-reload: + file.absent: + - name: /opt/so/state/addon_estemplates.txt + +# so-elasticsearch-templates-load will have its first successful run during the 'so-elastic-fleet-setup' script +so-elasticsearch-templates: + cmd.run: +{%- if GLOBALS.role == "so-heavynode" %} + - name: /usr/sbin/so-elasticsearch-templates-load --heavynode +{%- else %} + - name: /usr/sbin/so-elasticsearch-templates-load +{%- endif %} + - cwd: /opt/so + - template: jinja + - require: + - docker_container: so-elasticsearch + - file: elasticsearch_sbin_jinja + +so-elasticsearch-pipelines: + cmd.run: + - name: /usr/sbin/so-elasticsearch-pipelines {{ GLOBALS.hostname }} + - require: + - docker_container: so-elasticsearch + - file: so-elasticsearch-pipelines-script + +so-elasticsearch-roles-load: + cmd.run: + - name: /usr/sbin/so-elasticsearch-roles-load + - cwd: /opt/so + - template: jinja + - require: + - docker_container: so-elasticsearch + - file: elasticsearch_sbin_jinja + +{% if grains.role in ['so-managersearch', 'so-manager', 'so-managerhype'] %} +{% set ap = "absent" %} +{% endif %} +{% if grains.role in ['so-eval', 'so-standalone', 'so-heavynode'] %} +{% if ELASTICSEARCHMERGED.index_clean %} +{% set ap = "present" %} +{% else %} +{% set ap = "absent" %} +{% endif %} +{% endif %} +{% if grains.role in ['so-eval', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-manager'] %} +so-elasticsearch-indices-delete: + cron.{{ap}}: + - name: /usr/sbin/so-elasticsearch-indices-delete > /opt/so/log/elasticsearch/cron-elasticsearch-indices-delete.log 2>&1 + - identifier: so-elasticsearch-indices-delete + - user: root + - minute: '*/5' + - hour: '*' + - daymonth: '*' + - month: '*' + - dayweek: '*' +{% endif %} + +{% else %} + +{{sls}}_state_not_allowed: + test.fail_without_changes: + - name: {{sls}}_state_not_allowed + +{% endif %} diff --git a/salt/elasticsearch/config.sls b/salt/elasticsearch/config.sls index 41ef02164..8a4674c71 100644 --- a/salt/elasticsearch/config.sls +++ b/salt/elasticsearch/config.sls @@ -66,6 +66,8 @@ so-elasticsearch-ilm-policy-load-script: - group: 939 - mode: 754 - template: jinja + - defaults: + GLOBALS: {{ GLOBALS }} - show_changes: False so-elasticsearch-pipelines-script: @@ -91,6 +93,13 @@ estemplatedir: - group: 939 - makedirs: True +esaddontemplatedir: + file.directory: + - name: /opt/so/conf/elasticsearch/templates/addon-index + - user: 930 + - group: 939 + - makedirs: True + esrolesdir: file.directory: - name: /opt/so/conf/elasticsearch/roles diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index d0ab0f959..52964b9cf 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -1,6 +1,6 @@ elasticsearch: enabled: false - version: 9.0.8 + version: 9.3.3 index_clean: true vm: max_map_count: 1048576 @@ -3958,10 +3958,13 @@ elasticsearch: - vulnerability-mappings - common-settings - common-dynamic-mappings + - logs-redis.log@package + - logs-redis.log@custom data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: [] + ignore_missing_component_templates: + - logs-redis.log@custom index_patterns: - logs-redis.log* priority: 501 diff --git a/salt/elasticsearch/enabled.sls b/salt/elasticsearch/enabled.sls index 29ab80329..66d397b39 100644 --- a/salt/elasticsearch/enabled.sls +++ b/salt/elasticsearch/enabled.sls @@ -10,8 +10,6 @@ {% from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_NODES %} {% from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_SEED_HOSTS %} {% from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %} -{% set TEMPLATES = salt['pillar.get']('elasticsearch:templates', {}) %} -{% from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS %} include: - ca @@ -19,6 +17,9 @@ include: - elasticsearch.ssl - elasticsearch.config - elasticsearch.sostatus +{%- if GLOBALS.role != "so-searchnode" %} + - elasticsearch.cluster +{%- endif%} so-elasticsearch: docker_container.running: @@ -101,134 +102,24 @@ so-elasticsearch: - cmd: auth_users_roles_inode - cmd: auth_users_inode +wait_for_so-elasticsearch: + http.wait_for_successful_query: + - name: "https://localhost:9200/" + - username: 'so_elastic' + - password: '{{ ELASTICSEARCHMERGED.auth.users.so_elastic_user.pass }}' + - ssl: True + - verify_ssl: False + - status: 200 + - wait_for: 300 + - request_interval: 15 + - require: + - docker_container: so-elasticsearch + delete_so-elasticsearch_so-status.disabled: file.uncomment: - name: /opt/so/conf/so-status/so-status.conf - regex: ^so-elasticsearch$ -{% if GLOBALS.role != "so-searchnode" %} -escomponenttemplates: - file.recurse: - - name: /opt/so/conf/elasticsearch/templates/component - - source: salt://elasticsearch/templates/component - - user: 930 - - group: 939 - - clean: True - - onchanges_in: - - file: so-elasticsearch-templates-reload - - show_changes: False - -# Auto-generate templates from defaults file -{% for index, settings in ES_INDEX_SETTINGS.items() %} - {% if settings.index_template is defined %} -es_index_template_{{index}}: - file.managed: - - name: /opt/so/conf/elasticsearch/templates/index/{{ index }}-template.json - - source: salt://elasticsearch/base-template.json.jinja - - defaults: - TEMPLATE_CONFIG: {{ settings.index_template }} - - template: jinja - - show_changes: False - - onchanges_in: - - file: so-elasticsearch-templates-reload -{% endif %} -{% endfor %} - -{% if TEMPLATES %} -# Sync custom templates to /opt/so/conf/elasticsearch/templates -{% for TEMPLATE in TEMPLATES %} -es_template_{{TEMPLATE.split('.')[0] | replace("/","_") }}: - file.managed: - - source: salt://elasticsearch/templates/index/{{TEMPLATE}} -{% if 'jinja' in TEMPLATE.split('.')[-1] %} - - name: /opt/so/conf/elasticsearch/templates/index/{{TEMPLATE.split('/')[1] | replace(".jinja", "")}} - - template: jinja -{% else %} - - name: /opt/so/conf/elasticsearch/templates/index/{{TEMPLATE.split('/')[1]}} -{% endif %} - - user: 930 - - group: 939 - - show_changes: False - - onchanges_in: - - file: so-elasticsearch-templates-reload -{% endfor %} -{% endif %} - -{% if GLOBALS.role in GLOBALS.manager_roles %} -so-es-cluster-settings: - cmd.run: - - name: /usr/sbin/so-elasticsearch-cluster-settings - - cwd: /opt/so - - template: jinja - - require: - - docker_container: so-elasticsearch - - file: elasticsearch_sbin_jinja -{% endif %} - -so-elasticsearch-ilm-policy-load: - cmd.run: - - name: /usr/sbin/so-elasticsearch-ilm-policy-load - - cwd: /opt/so - - require: - - docker_container: so-elasticsearch - - file: so-elasticsearch-ilm-policy-load-script - - onchanges: - - file: so-elasticsearch-ilm-policy-load-script - -so-elasticsearch-templates-reload: - file.absent: - - name: /opt/so/state/estemplates.txt - -so-elasticsearch-templates: - cmd.run: - - name: /usr/sbin/so-elasticsearch-templates-load - - cwd: /opt/so - - template: jinja - - require: - - docker_container: so-elasticsearch - - file: elasticsearch_sbin_jinja - -so-elasticsearch-pipelines: - cmd.run: - - name: /usr/sbin/so-elasticsearch-pipelines {{ GLOBALS.hostname }} - - require: - - docker_container: so-elasticsearch - - file: so-elasticsearch-pipelines-script - -so-elasticsearch-roles-load: - cmd.run: - - name: /usr/sbin/so-elasticsearch-roles-load - - cwd: /opt/so - - template: jinja - - require: - - docker_container: so-elasticsearch - - file: elasticsearch_sbin_jinja - -{% if grains.role in ['so-managersearch', 'so-manager', 'so-managerhype'] %} -{% set ap = "absent" %} -{% endif %} -{% if grains.role in ['so-eval', 'so-standalone', 'so-heavynode'] %} -{% if ELASTICSEARCHMERGED.index_clean %} -{% set ap = "present" %} -{% else %} -{% set ap = "absent" %} -{% endif %} -{% endif %} -{% if grains.role in ['so-eval', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-manager'] %} -so-elasticsearch-indices-delete: - cron.{{ap}}: - - name: /usr/sbin/so-elasticsearch-indices-delete > /opt/so/log/elasticsearch/cron-elasticsearch-indices-delete.log 2>&1 - - identifier: so-elasticsearch-indices-delete - - user: root - - minute: '*/5' - - hour: '*' - - daymonth: '*' - - month: '*' - - dayweek: '*' -{% endif %} - -{% endif %} - {% else %} {{sls}}_state_not_allowed: diff --git a/salt/elasticsearch/files/ingest/common b/salt/elasticsearch/files/ingest/common index b7048cf3b..5923977c6 100644 --- a/salt/elasticsearch/files/ingest/common +++ b/salt/elasticsearch/files/ingest/common @@ -63,7 +63,8 @@ { "set": { "if": "ctx.event?.dataset != null && !ctx.event.dataset.contains('.')", "field": "event.dataset", "value": "{{event.module}}.{{event.dataset}}" } }, { "split": { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "separator": "\\.", "target_field": "dataset_tag_temp" } }, { "append": { "if": "ctx.dataset_tag_temp != null", "field": "tags", "value": "{{dataset_tag_temp.1}}" } }, - { "grok": { "if": "ctx.http?.response?.status_code != null", "field": "http.response.status_code", "patterns": ["%{NUMBER:http.response.status_code:long} %{GREEDYDATA}"]} }, + { "grok": { "if": "ctx.http?.response?.status_code instanceof String", "field": "http.response.status_code", "patterns": ["%{NUMBER:http.response.status_code:long}(?:\\s+%{GREEDYDATA})?"], "ignore_failure": true } }, + { "convert": { "if": "ctx.http?.response?.status_code != null && !(ctx.http.response.status_code instanceof Number)", "field": "http.response.status_code", "type": "long", "ignore_failure": true } }, { "set": { "if": "ctx?.metadata?.kafka != null" , "field": "kafka.id", "value": "{{metadata.kafka.partition}}{{metadata.kafka.offset}}{{metadata.kafka.timestamp}}", "ignore_failure": true } }, { "remove": { "field": [ "message2", "type", "fields", "category", "module", "dataset", "dataset_tag_temp", "event.dataset_temp" ], "ignore_missing": true, "ignore_failure": true } }, { "pipeline": { "name": "global@custom", "ignore_missing_pipeline": true, "description": "[Fleet] Global pipeline for all data streams" } } diff --git a/salt/elasticsearch/files/ingest/global@custom b/salt/elasticsearch/files/ingest/global@custom index bafb783a4..979c5c1b8 100644 --- a/salt/elasticsearch/files/ingest/global@custom +++ b/salt/elasticsearch/files/ingest/global@custom @@ -177,12 +177,84 @@ "description": "Extract IPs from Elastic Agent events (host.ip) and adds them to related.ip" } }, + { + "script": { + "description": "Snapshot event.ingested into _tmp.event_ingested_pre_fleet before .fleet_final_pipeline-1 overwrites it with ES ingest time", + "lang": "painless", + "if": "ctx.event?.ingested != null && ctx.event?.created == null", + "ignore_failure": true, + "source": "ctx.putIfAbsent('_tmp', [:]); ctx._tmp.event_ingested_pre_fleet = ctx.event.ingested;" + } + }, { "pipeline": { "name": ".fleet_final_pipeline-1", "ignore_missing_pipeline": true } }, + { + "script": { + "description": "Calculate time from Elastic Agent to Logstash.", + "lang": "painless", + "if": "ctx._tmp?.logstash_from_agent != null", + "ignore_failure": true, + "source": "ZonedDateTime start = ctx._tmp.event_ingested_pre_fleet != null ? ZonedDateTime.parse(ctx._tmp.event_ingested_pre_fleet) : ZonedDateTime.parse(ctx['@timestamp']); ctx.event.putIfAbsent('ingestion', [:]); ctx.event.ingestion.latency_elasticagent_to_logstash = ChronoUnit.SECONDS.between(start, ZonedDateTime.parse(ctx._tmp.logstash_from_agent));" + } + }, + { + "script": { + "description": "Calculate time from Logstash to Redis", + "lang": "painless", + "if": "ctx._tmp?.logstash_from_agent != null && ctx._tmp?.logstash_to_redis != null", + "ignore_failure": true, + "source": "ctx.event.putIfAbsent('ingestion', [:]); ctx.event.ingestion.latency_logstash_to_redis = ChronoUnit.SECONDS.between(ZonedDateTime.parse(ctx._tmp.logstash_from_agent), ZonedDateTime.parse(ctx._tmp.logstash_to_redis));" + } + }, + { + "script": { + "description": "Calculate time message spends in redis queue (logstash delay in pulling event).", + "lang": "painless", + "if": "ctx._tmp?.logstash_to_redis != null && ctx._tmp?.logstash_from_redis != null", + "ignore_failure": true, + "source": "ctx.event.putIfAbsent('ingestion', [:]); ctx.event.ingestion.latency_redis_to_logstash = ChronoUnit.SECONDS.between(ZonedDateTime.parse(ctx._tmp.logstash_to_redis), ZonedDateTime.parse(ctx._tmp.logstash_from_redis));" + } + }, + { + "script": { + "description": "Calculate time from Logstash to Elasticsearch (after read from Redis).", + "lang": "painless", + "if": "ctx._tmp?.logstash_from_redis != null", + "ignore_failure": true, + "source": "ctx.event.putIfAbsent('ingestion', [:]); ctx.event.ingestion.latency_logstash_to_elasticsearch = ChronoUnit.SECONDS.between(ZonedDateTime.parse(ctx._tmp.logstash_from_redis), metadata().now);" + } + }, + { + "script": { + "description": "Calculate time from Elastic Agent to Kafka.", + "lang": "painless", + "if": "ctx._tmp?.logstash_from_kafka != null && ctx._tmp?.logstash_from_agent == null", + "ignore_failure": true, + "source": "ZonedDateTime start = ctx._tmp.event_ingested_pre_fleet != null ? ZonedDateTime.parse(ctx._tmp.event_ingested_pre_fleet) : ZonedDateTime.parse(ctx['@timestamp']); ctx.event.putIfAbsent('ingestion', [:]); ctx.event.ingestion.latency_elasticagent_to_kafka = ChronoUnit.SECONDS.between(start, ZonedDateTime.parse(ctx._tmp.logstash_from_kafka));" + } + }, + { + "script": { + "description": "Calculate time message spends in Kafka queue (logstash delay in pulling event).", + "lang": "painless", + "if": "ctx._tmp?.logstash_from_kafka != null && ctx.metadata?.kafka?.timestamp != null && ctx._tmp?.logstash_from_agent == null", + "ignore_failure": true, + "source": "ctx.event.putIfAbsent('ingestion', [:]); ctx.event.ingestion.latency_kafka_queue = ChronoUnit.SECONDS.between(ZonedDateTime.ofInstant(Instant.ofEpochMilli(Long.parseLong(ctx.metadata.kafka.timestamp.toString())), ZoneId.of('UTC')), ZonedDateTime.parse(ctx._tmp.logstash_from_kafka));" + } + }, + { + "script": { + "description": "Calculate time from Logstash to Elasticsearch (after read from Kafka).", + "lang": "painless", + "if": "ctx._tmp?.logstash_from_kafka != null && ctx._tmp?.logstash_from_agent == null", + "ignore_failure": true, + "source": "ctx.event.putIfAbsent('ingestion', [:]); ctx.event.ingestion.latency_kafka_to_elasticsearch = ChronoUnit.SECONDS.between(ZonedDateTime.parse(ctx._tmp.logstash_from_kafka), metadata().now);" + } + }, { "remove": { "field": "event.agent_id_status", @@ -202,11 +274,12 @@ "event.dataset_temp", "dataset_tag_temp", "module_temp", - "datastream_dataset_temp" + "datastream_dataset_temp", + "_tmp" ], "ignore_missing": true, "ignore_failure": true } } ] -} \ No newline at end of file +} diff --git a/salt/elasticsearch/files/ingest/logs-pfsense.log-1.23.1 b/salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.2 similarity index 72% rename from salt/elasticsearch/files/ingest/logs-pfsense.log-1.23.1 rename to salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.2 index d3354f363..1ea828514 100644 --- a/salt/elasticsearch/files/ingest/logs-pfsense.log-1.23.1 +++ b/salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.2 @@ -10,24 +10,28 @@ "processors": [ { "set": { + "tag": "set_ecs_version_f5923549", "field": "ecs.version", "value": "8.17.0" } }, { "set": { + "tag": "set_observer_vendor_ad9d35cc", "field": "observer.vendor", "value": "netgate" } }, { "set": { + "tag": "set_observer_type_5dddf3ba", "field": "observer.type", "value": "firewall" } }, { "rename": { + "tag": "rename_message_to_event_original_56a77271", "field": "message", "target_field": "event.original", "ignore_missing": true, @@ -36,12 +40,14 @@ }, { "set": { + "tag": "set_event_kind_de80643c", "field": "event.kind", "value": "event" } }, { "set": { + "tag": "set_event_timezone_4ca44cac", "field": "event.timezone", "value": "{{{_tmp.tz_offset}}}", "if": "ctx._tmp?.tz_offset != null && ctx._tmp?.tz_offset != 'local'" @@ -49,6 +55,7 @@ }, { "grok": { + "tag": "grok_event_original_27d9c8c7", "description": "Parse syslog header", "field": "event.original", "patterns": [ @@ -72,6 +79,7 @@ }, { "date": { + "tag": "date__tmp_timestamp8601_to_timestamp_6ac9d3ce", "if": "ctx._tmp.timestamp8601 != null", "field": "_tmp.timestamp8601", "target_field": "@timestamp", @@ -82,6 +90,7 @@ }, { "date": { + "tag": "date__tmp_timestamp_to_timestamp_f21e536e", "if": "ctx.event?.timezone != null && ctx._tmp?.timestamp != null", "field": "_tmp.timestamp", "target_field": "@timestamp", @@ -95,6 +104,7 @@ }, { "grok": { + "tag": "grok_process_name_cef3d489", "description": "Set Event Provider", "field": "process.name", "patterns": [ @@ -107,71 +117,83 @@ }, { "pipeline": { - "name": "logs-pfsense.log-1.23.1-firewall", + "tag": "pipeline_e16851a7", + "name": "logs-pfsense.log-1.25.2-firewall", "if": "ctx.event.provider == 'filterlog'" } }, { "pipeline": { - "name": "logs-pfsense.log-1.23.1-openvpn", + "tag": "pipeline_828590b5", + "name": "logs-pfsense.log-1.25.2-openvpn", "if": "ctx.event.provider == 'openvpn'" } }, { "pipeline": { - "name": "logs-pfsense.log-1.23.1-ipsec", + "tag": "pipeline_9d37039c", + "name": "logs-pfsense.log-1.25.2-ipsec", "if": "ctx.event.provider == 'charon'" } }, { "pipeline": { - "name": "logs-pfsense.log-1.23.1-dhcp", - "if": "[\"dhcpd\", \"dhclient\", \"dhcp6c\"].contains(ctx.event.provider)" + "tag": "pipeline_ad56bbca", + "name": "logs-pfsense.log-1.25.2-dhcp", + "if": "[\"dhcpd\", \"dhclient\", \"dhcp6c\", \"dnsmasq-dhcp\"].contains(ctx.event.provider)" } }, { "pipeline": { - "name": "logs-pfsense.log-1.23.1-unbound", + "tag": "pipeline_dd85553d", + "name": "logs-pfsense.log-1.25.2-unbound", "if": "ctx.event.provider == 'unbound'" } }, { "pipeline": { - "name": "logs-pfsense.log-1.23.1-haproxy", + "tag": "pipeline_720ed255", + "name": "logs-pfsense.log-1.25.2-haproxy", "if": "ctx.event.provider == 'haproxy'" } }, { "pipeline": { - "name": "logs-pfsense.log-1.23.1-php-fpm", + "tag": "pipeline_456beba5", + "name": "logs-pfsense.log-1.25.2-php-fpm", "if": "ctx.event.provider == 'php-fpm'" } }, { "pipeline": { - "name": "logs-pfsense.log-1.23.1-squid", + "tag": "pipeline_a0d89375", + "name": "logs-pfsense.log-1.25.2-squid", "if": "ctx.event.provider == 'squid'" } }, { "pipeline": { - "name": "logs-pfsense.log-1.23.1-snort", + "tag": "pipeline_c2f1ed55", + "name": "logs-pfsense.log-1.25.2-snort", "if": "ctx.event.provider == 'snort'" } }, { "pipeline": { - "name": "logs-pfsense.log-1.23.1-suricata", + "tag":"pipeline_33db1c9e", + "name": "logs-pfsense.log-1.25.2-suricata", "if": "ctx.event.provider == 'suricata'" } }, { "drop": { - "if": "![\"filterlog\", \"openvpn\", \"charon\", \"dhcpd\", \"dhclient\", \"dhcp6c\", \"unbound\", \"haproxy\", \"php-fpm\", \"squid\", \"snort\", \"suricata\"].contains(ctx.event?.provider)" + "tag": "drop_9d7c46f8", + "if": "![\"filterlog\", \"openvpn\", \"charon\", \"dhcpd\", \"dnsmasq-dhcp\", \"dhclient\", \"dhcp6c\", \"unbound\", \"haproxy\", \"php-fpm\", \"squid\", \"snort\", \"suricata\"].contains(ctx.event?.provider)" } }, { "append": { + "tag": "append_event_category_4780a983", "field": "event.category", "value": "network", "if": "ctx.network != null" @@ -179,6 +201,7 @@ }, { "convert": { + "tag": "convert_source_address_to_source_ip_f5632a20", "field": "source.address", "target_field": "source.ip", "type": "ip", @@ -188,6 +211,7 @@ }, { "convert": { + "tag": "convert_destination_address_to_destination_ip_f1388f0c", "field": "destination.address", "target_field": "destination.ip", "type": "ip", @@ -197,6 +221,7 @@ }, { "set": { + "tag": "set_network_type_1f1d940a", "field": "network.type", "value": "ipv6", "if": "ctx.source?.ip != null && ctx.source.ip.contains(\":\")" @@ -204,6 +229,7 @@ }, { "set": { + "tag": "set_network_type_69deca38", "field": "network.type", "value": "ipv4", "if": "ctx.source?.ip != null && ctx.source.ip.contains(\".\")" @@ -211,6 +237,7 @@ }, { "geoip": { + "tag": "geoip_source_ip_to_source_geo_da2e41b2", "field": "source.ip", "target_field": "source.geo", "ignore_missing": true @@ -218,6 +245,7 @@ }, { "geoip": { + "tag": "geoip_destination_ip_to_destination_geo_ab5e2968", "field": "destination.ip", "target_field": "destination.geo", "ignore_missing": true @@ -225,6 +253,7 @@ }, { "geoip": { + "tag": "geoip_source_ip_to_source_as_28d69883", "ignore_missing": true, "database_file": "GeoLite2-ASN.mmdb", "field": "source.ip", @@ -237,6 +266,7 @@ }, { "geoip": { + "tag": "geoip_destination_ip_to_destination_as_8a007787", "database_file": "GeoLite2-ASN.mmdb", "field": "destination.ip", "target_field": "destination.as", @@ -249,6 +279,7 @@ }, { "rename": { + "tag": "rename_source_as_asn_to_source_as_number_a917047d", "field": "source.as.asn", "target_field": "source.as.number", "ignore_missing": true @@ -256,6 +287,7 @@ }, { "rename": { + "tag": "rename_source_as_organization_name_to_source_as_organization_name_f1362d0b", "field": "source.as.organization_name", "target_field": "source.as.organization.name", "ignore_missing": true @@ -263,6 +295,7 @@ }, { "rename": { + "tag": "rename_destination_as_asn_to_destination_as_number_3b459fcd", "field": "destination.as.asn", "target_field": "destination.as.number", "ignore_missing": true @@ -270,6 +303,7 @@ }, { "rename": { + "tag": "rename_destination_as_organization_name_to_destination_as_organization_name_814bd459", "field": "destination.as.organization_name", "target_field": "destination.as.organization.name", "ignore_missing": true @@ -277,12 +311,14 @@ }, { "community_id": { + "tag": "community_id_d2308e7a", "target_field": "network.community_id", "ignore_failure": true } }, { "grok": { + "tag": "grok_observer_ingress_interface_name_968018d3", "field": "observer.ingress.interface.name", "patterns": [ "%{DATA}.%{NONNEGINT:observer.ingress.vlan.id}" @@ -293,6 +329,7 @@ }, { "set": { + "tag": "set_network_vlan_id_efd4d96a", "field": "network.vlan.id", "copy_from": "observer.ingress.vlan.id", "ignore_empty_value": true @@ -300,6 +337,7 @@ }, { "append": { + "tag": "append_related_ip_c1a6356b", "field": "related.ip", "value": "{{{destination.ip}}}", "allow_duplicates": false, @@ -308,6 +346,7 @@ }, { "append": { + "tag": "append_related_ip_8121c591", "field": "related.ip", "value": "{{{source.ip}}}", "allow_duplicates": false, @@ -316,6 +355,7 @@ }, { "append": { + "tag": "append_related_ip_53b62ed8", "field": "related.ip", "value": "{{{source.nat.ip}}}", "allow_duplicates": false, @@ -324,6 +364,7 @@ }, { "append": { + "tag": "append_related_hosts_6f162628", "field": "related.hosts", "value": "{{{destination.domain}}}", "if": "ctx.destination?.domain != null" @@ -331,6 +372,7 @@ }, { "append": { + "tag": "append_related_user_c036eec2", "field": "related.user", "value": "{{{user.name}}}", "if": "ctx.user?.name != null" @@ -338,6 +380,7 @@ }, { "set": { + "tag": "set_network_direction_cb1e3125", "field": "network.direction", "value": "{{{network.direction}}}bound", "if": "ctx.network?.direction != null && ctx.network?.direction =~ /^(in|out)$/" @@ -345,6 +388,7 @@ }, { "remove": { + "tag": "remove_a82e20f2", "field": [ "_tmp" ], @@ -353,11 +397,21 @@ }, { "script": { + "tag": "script_a7f2c062", "lang": "painless", "description": "This script processor iterates over the whole document to remove fields with null values.", "source": "void handleMap(Map map) {\n for (def x : map.values()) {\n if (x instanceof Map) {\n handleMap(x);\n } else if (x instanceof List) {\n handleList(x);\n }\n }\n map.values().removeIf(v -> v == null || (v instanceof String && v == \"-\"));\n}\nvoid handleList(List list) {\n for (def x : list) {\n if (x instanceof Map) {\n handleMap(x);\n } else if (x instanceof List) {\n handleList(x);\n }\n }\n}\nhandleMap(ctx);\n" } }, + { + "append": { + "tag": "append_preserve_original_event_on_error", + "field": "tags", + "value": "preserve_original_event", + "allow_duplicates": false, + "if": "ctx.error?.message != null" + } + }, { "pipeline": { "name": "global@custom", @@ -405,7 +459,14 @@ { "append": { "field": "error.message", - "value": "{{{ _ingest.on_failure_message }}}" + "value": "Processor '{{{ _ingest.on_failure_processor_type }}}' {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}'" + } + }, + { + "append": { + "field": "tags", + "value": "preserve_original_event", + "allow_duplicates": false } } ] diff --git a/salt/elasticsearch/files/ingest/logs-pfsense.log-1.23.1-suricata b/salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.2-suricata similarity index 100% rename from salt/elasticsearch/files/ingest/logs-pfsense.log-1.23.1-suricata rename to salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.2-suricata diff --git a/salt/elasticsearch/files/ingest/zeek.ja4d b/salt/elasticsearch/files/ingest/zeek.ja4d new file mode 100644 index 000000000..206622c49 --- /dev/null +++ b/salt/elasticsearch/files/ingest/zeek.ja4d @@ -0,0 +1,71 @@ +{ + "description": "zeek.ja4d", + "processors": [ + { + "set": { + "field": "event.dataset", + "value": "ja4d" + } + }, + { + "remove": { + "field": [ + "host" + ], + "ignore_failure": true + } + }, + { + "json": { + "field": "message", + "target_field": "message2", + "ignore_failure": true + } + }, + { + "rename": { + "field": "message2.ja4d", + "target_field": "hash.ja4d", + "ignore_missing": true, + "if": "ctx?.message2?.ja4d != null && ctx.message2.ja4d.length() > 0" + } + }, + { + "rename": { + "field": "message2.client_mac", + "target_field": "host.mac", + "ignore_missing": true, + "if": "ctx?.message2?.client_mac != null && ctx.message2.client_mac.length() > 0" + } + }, + { + "rename": { + "field": "message2.hostname", + "target_field": "host.hostname", + "ignore_missing": true, + "if": "ctx?.message2?.hostname != null && ctx.message2.hostname.length() > 0" + } + }, + { + "rename": { + "field": "message2.requested_ip", + "target_field": "dhcp.requested_address", + "ignore_missing": true, + "if": "ctx?.message2?.requested_ip != null && ctx.message2.requested_ip.length() > 0" + } + }, + { + "rename": { + "field": "message2.vendor_class_id", + "target_field": "zeek.ja4d.vendor_class_id", + "ignore_missing": true, + "if": "ctx?.message2?.vendor_class_id != null && ctx.message2.vendor_class_id.length() > 0" + } + }, + { + "pipeline": { + "name": "zeek.common" + } + } + ] +} \ No newline at end of file diff --git a/salt/elasticsearch/files/log4j2.properties b/salt/elasticsearch/files/log4j2.properties index b29378d6a..050071581 100644 --- a/salt/elasticsearch/files/log4j2.properties +++ b/salt/elasticsearch/files/log4j2.properties @@ -45,3 +45,7 @@ appender.rolling_json.strategy.action.condition.nested_condition.age = 1D rootLogger.level = info rootLogger.appenderRef.rolling.ref = rolling rootLogger.appenderRef.rolling_json.ref = rolling_json + +# Suppress NotEntitledException WARNs (ES 9.3.3 bug) +logger.entitlement_security.name = org.elasticsearch.entitlement.runtime.policy.PolicyManager.x-pack-security.org.elasticsearch.security.org.elasticsearch.xpack.security +logger.entitlement_security.level = error \ No newline at end of file diff --git a/salt/elasticsearch/template.map.jinja b/salt/elasticsearch/template.map.jinja index 2563f8e23..e66057775 100644 --- a/salt/elasticsearch/template.map.jinja +++ b/salt/elasticsearch/template.map.jinja @@ -14,15 +14,42 @@ {% set ES_INDEX_SETTINGS_ORIG = ELASTICSEARCHDEFAULTS.elasticsearch.index_settings %} +{% set ALL_ADDON_INTEGRATION_DEFAULTS = {} %} +{% set ALL_ADDON_SETTINGS_ORIG = {} %} +{% set ALL_ADDON_SETTINGS_GLOBAL_OVERRIDES = {} %} +{% set ALL_ADDON_SETTINGS = {} %} {# start generation of integration default index_settings #} -{% if salt['file.file_exists']('/opt/so/state/esfleet_package_components.json') and salt['file.file_exists']('/opt/so/state/esfleet_component_templates.json') %} -{% set check_package_components = salt['file.stats']('/opt/so/state/esfleet_package_components.json') %} -{% if check_package_components.size > 1 %} -{% from 'elasticfleet/integration-defaults.map.jinja' import ADDON_INTEGRATION_DEFAULTS %} -{% for index, settings in ADDON_INTEGRATION_DEFAULTS.items() %} -{% do ES_INDEX_SETTINGS_ORIG.update({index: settings}) %} -{% endfor %} -{% endif%} +{% if salt['file.file_exists']('/opt/so/state/esfleet_component_templates.json') %} +{# import integration type defaults #} +{% if salt['file.file_exists']('/opt/so/state/esfleet_package_components.json') %} +{% set check_integration_package_components = salt['file.stats']('/opt/so/state/esfleet_package_components.json') %} +{% if check_integration_package_components.size > 1 %} +{% from 'elasticfleet/integration-defaults.map.jinja' import ADDON_INTEGRATION_DEFAULTS %} +{% do ALL_ADDON_INTEGRATION_DEFAULTS.update(ADDON_INTEGRATION_DEFAULTS) %} +{% endif %} +{% endif %} + +{# import input type defaults #} +{% if salt['file.file_exists']('/opt/so/state/esfleet_input_package_components.json') %} +{% set check_input_package_components = salt['file.stats']('/opt/so/state/esfleet_input_package_components.json') %} +{% if check_input_package_components.size > 1 %} +{% from 'elasticfleet/input-defaults.map.jinja' import ADDON_INPUT_INTEGRATION_DEFAULTS %} +{% do ALL_ADDON_INTEGRATION_DEFAULTS.update(ADDON_INPUT_INTEGRATION_DEFAULTS) %} +{% endif %} +{% endif %} + +{# import content type defaults #} +{% if salt['file.file_exists']('/opt/so/state/esfleet_content_package_components.json') %} +{% set check_content_package_components = salt['file.stats']('/opt/so/state/esfleet_content_package_components.json') %} +{% if check_content_package_components.size > 1 %} +{% from 'elasticfleet/content-defaults.map.jinja' import ADDON_CONTENT_INTEGRATION_DEFAULTS %} +{% do ALL_ADDON_INTEGRATION_DEFAULTS.update(ADDON_CONTENT_INTEGRATION_DEFAULTS) %} +{% endif %} +{% endif %} + +{% for index, settings in ALL_ADDON_INTEGRATION_DEFAULTS.items() %} +{% do ALL_ADDON_SETTINGS_ORIG.update({index: settings}) %} +{% endfor %} {% endif %} {# end generation of integration default index_settings #} @@ -31,25 +58,33 @@ {% do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.update({index: salt['defaults.merge'](ELASTICSEARCHDEFAULTS.elasticsearch.index_settings[index], PILLAR_GLOBAL_OVERRIDES, in_place=False)}) %} {% endfor %} +{% if ALL_ADDON_SETTINGS_ORIG.keys() | length > 0 %} +{% for index in ALL_ADDON_SETTINGS_ORIG.keys() %} +{% do ALL_ADDON_SETTINGS_GLOBAL_OVERRIDES.update({index: salt['defaults.merge'](ALL_ADDON_SETTINGS_ORIG[index], PILLAR_GLOBAL_OVERRIDES, in_place=False)}) %} +{% endfor %} +{% endif %} + {% set ES_INDEX_SETTINGS = {} %} -{% do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.update(salt['defaults.merge'](ES_INDEX_SETTINGS_GLOBAL_OVERRIDES, ES_INDEX_PILLAR, in_place=False)) %} -{% for index, settings in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.items() %} +{% macro create_final_index_template(DEFINED_SETTINGS, GLOBAL_OVERRIDES, FINAL_INDEX_SETTINGS) %} + +{% do GLOBAL_OVERRIDES.update(salt['defaults.merge'](GLOBAL_OVERRIDES, ES_INDEX_PILLAR, in_place=False)) %} +{% for index, settings in GLOBAL_OVERRIDES.items() %} {# prevent this action from being performed on custom defined indices. #} {# the custom defined index is not present in either of the dictionaries and fails to reder. #} -{% if index in ES_INDEX_SETTINGS_ORIG and index in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES %} +{% if index in DEFINED_SETTINGS and index in GLOBAL_OVERRIDES %} {# dont merge policy from the global_overrides if policy isn't defined in the original index settingss #} {# this will prevent so-elasticsearch-ilm-policy-load from trying to load policy on non ILM manged indices #} -{% if not ES_INDEX_SETTINGS_ORIG[index].policy is defined and ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy is defined %} -{% do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].pop('policy') %} +{% if not DEFINED_SETTINGS[index].policy is defined and GLOBAL_OVERRIDES[index].policy is defined %} +{% do GLOBAL_OVERRIDES[index].pop('policy') %} {% endif %} {# this prevents and index from inderiting a policy phase from global overrides if it wasnt defined in the defaults. #} -{% if ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy is defined %} -{% for phase in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy.phases.copy() %} -{% if ES_INDEX_SETTINGS_ORIG[index].policy.phases[phase] is not defined %} -{% do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy.phases.pop(phase) %} +{% if GLOBAL_OVERRIDES[index].policy is defined %} +{% for phase in GLOBAL_OVERRIDES[index].policy.phases.copy() %} +{% if DEFINED_SETTINGS[index].policy.phases[phase] is not defined %} +{% do GLOBAL_OVERRIDES[index].policy.phases.pop(phase) %} {% endif %} {% endfor %} {% endif %} @@ -111,5 +146,14 @@ {% endfor %} {% endif %} -{% do ES_INDEX_SETTINGS.update({index | replace("_x_", "."): ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index]}) %} +{% do FINAL_INDEX_SETTINGS.update({index | replace("_x_", "."): GLOBAL_OVERRIDES[index]}) %} {% endfor %} +{% endmacro %} + +{{ create_final_index_template(ES_INDEX_SETTINGS_ORIG, ES_INDEX_SETTINGS_GLOBAL_OVERRIDES, ES_INDEX_SETTINGS) }} +{{ create_final_index_template(ALL_ADDON_SETTINGS_ORIG, ALL_ADDON_SETTINGS_GLOBAL_OVERRIDES, ALL_ADDON_SETTINGS) }} + +{% set SO_MANAGED_INDICES = [] %} +{% for index, settings in ES_INDEX_SETTINGS.items() %} +{% do SO_MANAGED_INDICES.append(index) %} +{% endfor %} \ No newline at end of file diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-component-templates-list b/salt/elasticsearch/tools/sbin/so-elasticsearch-component-templates-list index 2fccce9cb..6946e30da 100755 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-component-templates-list +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-component-templates-list @@ -6,8 +6,19 @@ # Elastic License 2.0. . /usr/sbin/so-common -if [ "$1" == "" ]; then - curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_component_template | jq '.component_templates[] |.name'| sort + +if [[ -z "$1" ]]; then + if output=$(so-elasticsearch-query "_component_template" --retry 3 --retry-delay 1 --fail); then + jq '[.component_templates[] | .name] | sort' <<< "$output" + else + echo "Failed to retrieve component templates from Elasticsearch." + exit 1 + fi else - curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_component_template/$1 | jq -fi + if output=$(so-elasticsearch-query "_component_template/$1" --retry 3 --retry-delay 1 --fail); then + jq <<< "$output" + else + echo "Failed to retrieve component template '$1' from Elasticsearch." + exit 1 + fi +fi \ No newline at end of file diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load b/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load new file mode 100755 index 000000000..a0ebd66e8 --- /dev/null +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load @@ -0,0 +1,276 @@ +#!/bin/bash +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +. /usr/sbin/so-common + +SO_STATEFILE_SUCCESS=/opt/so/state/estemplates.txt +ADDON_STATEFILE_SUCCESS=/opt/so/state/addon_estemplates.txt +ELASTICSEARCH_TEMPLATES_DIR="/opt/so/conf/elasticsearch/templates" +SO_TEMPLATES_DIR="${ELASTICSEARCH_TEMPLATES_DIR}/index" +ADDON_TEMPLATES_DIR="${ELASTICSEARCH_TEMPLATES_DIR}/addon-index" +SO_LOAD_FAILURES=0 +ADDON_LOAD_FAILURES=0 +SO_LOAD_FAILURES_NAMES=() +ADDON_LOAD_FAILURES_NAMES=() +IS_HEAVYNODE="false" +FORCE="false" +VERBOSE="false" +SHOULD_EXIT_ON_FAILURE="true" + +# If soup is running, ignore errors +pgrep soup >/dev/null && SHOULD_EXIT_ON_FAILURE="false" + +while [[ $# -gt 0 ]]; do + case "$1" in + --heavynode) + IS_HEAVYNODE="true" + ;; + --force) + FORCE="true" + ;; + --verbose) + VERBOSE="true" + ;; + *) + echo "Usage: $0 [options]" + echo "Options:" + echo " --heavynode Only loads index templates specific to heavynodes" + echo " --force Force reload all templates regardless of statefiles (default: false)" + echo " --verbose Enable verbose output" + exit 1 + ;; + esac + shift +done + +load_template() { + local uri="$1" + local file="$2" + + echo "Loading template file $file" + if ! output=$(retry 3 3 "so-elasticsearch-query $uri -d@$file -XPUT" "{\"acknowledged\":true}"); then + echo "$output" + + return 1 + + elif [[ "$VERBOSE" == "true" ]]; then + echo "$output" + fi + +} + +check_required_component_template_exists() { + local required + local missing + local file=$1 + + required=$(jq '[((.composed_of //[]) - (.ignore_missing_component_templates // []))[]]' "$file") + missing=$(jq -n --argjson required "$required" --argjson component_templates "$component_templates" '(($required) - ($component_templates))') + + if [[ $(jq length <<<"$missing") -gt 0 ]]; then + + return 1 + fi +} + +check_heavynode_compatiable_index_template() { + # The only templates that are relevant to heavynodes are from datasets defined in elasticagent/files/elastic-agent.yml.jinja. + # Heavynodes do not have fleet server packages installed and do not support elastic agents reporting directly to them. + local -A heavynode_index_templates=( + ["so-import"]=1 + ["so-syslog"]=1 + ["so-logs-soc"]=1 + ["so-suricata"]=1 + ["so-suricata.alerts"]=1 + ["so-zeek"]=1 + ["so-strelka"]=1 + ) + + local template_name="$1" + + if [[ ! -v heavynode_index_templates["$template_name"] ]]; then + + return 1 + fi + +} + +load_component_templates() { + local printed_name="$1" + local pattern="${ELASTICSEARCH_TEMPLATES_DIR}/component/$2" + local append_mappings="${3:-"false"}" + + echo -e "\nLoading $printed_name component templates...\n" + + if ! compgen -G "${pattern}/*.json" > /dev/null; then + echo "No $printed_name component templates found in ${pattern}, skipping." + return + fi + + for component in "$pattern"/*.json; do + tmpl_name=$(basename "${component%.json}") + + if [[ "$append_mappings" == "true" ]]; then + # avoid duplicating "-mappings" if it already exists in the component template filename + tmpl_name="${tmpl_name%-mappings}-mappings" + fi + + if ! load_template "_component_template/${tmpl_name}" "$component"; then + SO_LOAD_FAILURES=$((SO_LOAD_FAILURES + 1)) + SO_LOAD_FAILURES_NAMES+=("$component") + fi + done +} + +check_elasticsearch_responsive() { + # Cannot load templates if Elasticsearch is not responding. + # NOTE: Slightly faster exit w/ failure than previous "retry 240 1" if there is a problem with Elasticsearch the + # script should exit sooner rather than hang at the 'so-elasticsearch-templates' salt state. + retry 3 15 "so-elasticsearch-query / --output /dev/null --fail" || + fail "Elasticsearch is not responding. Please review Elasticsearch logs /opt/so/log/elasticsearch/securityonion.log for more details. Additionally, consider running so-elasticsearch-troubleshoot." +} + +index_templates_exist() { + local templates_dir="$1" + + if [[ ! -d "$templates_dir" ]]; then + return 1 + fi + + compgen -G "${templates_dir}/*.json" > /dev/null +} + +should_load_addon_templates() { + if [[ "$IS_HEAVYNODE" == "true" ]]; then + return 1 + fi + + # Skip statefile checks when forcing template load + if [[ "$FORCE" != "true" ]]; then + if [[ ! -f "$SO_STATEFILE_SUCCESS" || -f "$ADDON_STATEFILE_SUCCESS" ]]; then + return 1 + fi + fi + + index_templates_exist "$ADDON_TEMPLATES_DIR" +} + +if [[ "$FORCE" == "true" || ! -f "$SO_STATEFILE_SUCCESS" ]] && index_templates_exist "$SO_TEMPLATES_DIR"; then + check_elasticsearch_responsive + + if [[ "$IS_HEAVYNODE" == "false" ]]; then + # TODO: Better way to check if fleet server is installed vs checking for Elastic Defend component template. + fleet_check="logs-endpoint.alerts@package" + if ! so-elasticsearch-query "_component_template/$fleet_check" --output /dev/null --retry 5 --retry-delay 3 --fail; then + # This check prevents so-elasticsearch-templates-load from running before so-elastic-fleet-setup has run. + echo -e "\nPackage $fleet_check not yet installed. Fleet Server may not be fully configured yet." + # Fleet Server is required because some SO index templates depend on components installed via + # specific integrations eg Elastic Defend. These are components that we do not manually create / manage + # via /opt/so/saltstack/salt/elasticsearch/templates/component/ + + exit 0 + fi + fi + + # load_component_templates "Name" "directory" "append '-mappings'?" + load_component_templates "ECS" "ecs" "true" + load_component_templates "Elastic Agent" "elastic-agent" + load_component_templates "Security Onion" "so" + + component_templates=$(so-elasticsearch-component-templates-list) + echo -e "Loading Security Onion index templates...\n" + for so_idx_tmpl in "${SO_TEMPLATES_DIR}"/*.json; do + tmpl_name=$(basename "${so_idx_tmpl%-template.json}") + + if [[ "$IS_HEAVYNODE" == "true" ]]; then + # TODO: Better way to load only heavynode specific templates + if ! check_heavynode_compatiable_index_template "$tmpl_name"; then + if [[ "$VERBOSE" == "true" ]]; then + echo "Skipping over $so_idx_tmpl, template is not a heavynode specific index template." + fi + + continue + fi + fi + + if check_required_component_template_exists "$so_idx_tmpl"; then + if ! load_template "_index_template/$tmpl_name" "$so_idx_tmpl"; then + SO_LOAD_FAILURES=$((SO_LOAD_FAILURES + 1)) + SO_LOAD_FAILURES_NAMES+=("$so_idx_tmpl") + fi + else + echo "Skipping over $so_idx_tmpl due to missing required component template(s)." + SO_LOAD_FAILURES=$((SO_LOAD_FAILURES + 1)) + SO_LOAD_FAILURES_NAMES+=("$so_idx_tmpl") + + continue + fi + done + + if [[ $SO_LOAD_FAILURES -eq 0 ]]; then + echo "All Security Onion core templates loaded successfully." + + touch "$SO_STATEFILE_SUCCESS" + else + echo "Encountered $SO_LOAD_FAILURES failure(s) loading templates:" + for failed_template in "${SO_LOAD_FAILURES_NAMES[@]}"; do + echo " - $failed_template" + done + if [[ "$SHOULD_EXIT_ON_FAILURE" == "true" ]]; then + fail "Failed to load all Security Onion core templates successfully." + fi + fi +elif ! index_templates_exist "$SO_TEMPLATES_DIR"; then + echo "No Security Onion core index templates found in ${SO_TEMPLATES_DIR}, skipping." +elif [[ -f "$SO_STATEFILE_SUCCESS" ]]; then + echo "Security Onion core templates already loaded" +fi + +# Start loading addon templates +if should_load_addon_templates; then + + check_elasticsearch_responsive + + echo -e "\nLoading addon integration index templates...\n" + component_templates=$(so-elasticsearch-component-templates-list) + + for addon_idx_tmpl in "${ADDON_TEMPLATES_DIR}"/*.json; do + tmpl_name=$(basename "${addon_idx_tmpl%-template.json}") + + if check_required_component_template_exists "$addon_idx_tmpl"; then + if ! load_template "_index_template/${tmpl_name}" "$addon_idx_tmpl"; then + ADDON_LOAD_FAILURES=$((ADDON_LOAD_FAILURES + 1)) + ADDON_LOAD_FAILURES_NAMES+=("$addon_idx_tmpl") + fi + else + echo "Skipping over $addon_idx_tmpl due to missing required component template(s)." + ADDON_LOAD_FAILURES=$((ADDON_LOAD_FAILURES + 1)) + ADDON_LOAD_FAILURES_NAMES+=("$addon_idx_tmpl") + + continue + fi + done + + if [[ $ADDON_LOAD_FAILURES -eq 0 ]]; then + echo "All addon integration templates loaded successfully." + + touch "$ADDON_STATEFILE_SUCCESS" + else + echo "Encountered $ADDON_LOAD_FAILURES failure(s) loading addon integration templates:" + for failed_template in "${ADDON_LOAD_FAILURES_NAMES[@]}"; do + echo " - $failed_template" + done + if [[ "$SHOULD_EXIT_ON_FAILURE" == "true" ]]; then + fail "Failed to load all addon integration templates successfully." + fi + fi + +elif [[ ! -f "$SO_STATEFILE_SUCCESS" && "$IS_HEAVYNODE" == "false" ]]; then + echo "Skipping loading addon integration templates until Security Onion core templates have been loaded." + +elif [[ -f "$ADDON_STATEFILE_SUCCESS" && "$IS_HEAVYNODE" == "false" && "$FORCE" == "false" ]]; then + echo "Addon integration templates already loaded" +fi diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load index 04a7a8ab0..7988c1905 100755 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load @@ -7,6 +7,9 @@ . /usr/sbin/so-common {%- from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS %} +{%- if GLOBALS.role != "so-heavynode" %} +{%- from 'elasticsearch/template.map.jinja' import ALL_ADDON_SETTINGS %} +{%- endif %} {%- for index, settings in ES_INDEX_SETTINGS.items() %} {%- if settings.policy is defined %} @@ -33,3 +36,13 @@ {%- endif %} {%- endfor %} echo +{%- if GLOBALS.role != "so-heavynode" %} +{%- for index, settings in ALL_ADDON_SETTINGS.items() %} +{%- if settings.policy is defined %} + echo + echo "Setting up {{ index }}-logs policy..." + curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://localhost:9200/_ilm/policy/{{ index }}-logs" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }' + echo +{%- endif %} +{%- endfor %} +{%- endif %} diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load deleted file mode 100755 index ad3fe1344..000000000 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load +++ /dev/null @@ -1,165 +0,0 @@ -#!/bin/bash -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. -{%- import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %} -{% from 'vars/globals.map.jinja' import GLOBALS %} - -STATE_FILE_INITIAL=/opt/so/state/estemplates_initial_load_attempt.txt -STATE_FILE_SUCCESS=/opt/so/state/estemplates.txt - -if [[ -f $STATE_FILE_INITIAL ]]; then - # The initial template load has already run. As this is a subsequent load, all dependencies should - # already be satisified. Therefore, immediately exit/abort this script upon any template load failure - # since this is an unrecoverable failure. - should_exit_on_failure=1 -else - # This is the initial template load, and there likely are some components not yet setup in Elasticsearch. - # Therefore load as many templates as possible at this time and if an error occurs proceed to the next - # template. But if at least one template fails to load do not mark the templates as having been loaded. - # This will allow the next load to resume the load of the templates that failed to load initially. - should_exit_on_failure=0 - echo "This is the initial template load" -fi - -# If soup is running, ignore errors -pgrep soup > /dev/null && should_exit_on_failure=0 - -load_failures=0 - -load_template() { - uri=$1 - file=$2 - - echo "Loading template file $i" - if ! retry 3 1 "so-elasticsearch-query $uri -d@$file -XPUT" "{\"acknowledged\":true}"; then - if [[ $should_exit_on_failure -eq 1 ]]; then - fail "Could not load template file: $file" - else - load_failures=$((load_failures+1)) - echo "Incremented load failure counter: $load_failures" - fi - fi -} - -if [ ! -f $STATE_FILE_SUCCESS ]; then - echo "State file $STATE_FILE_SUCCESS not found. Running so-elasticsearch-templates-load." - - . /usr/sbin/so-common - - {% if GLOBALS.role != 'so-heavynode' %} - if [ -f /usr/sbin/so-elastic-fleet-common ]; then - . /usr/sbin/so-elastic-fleet-common - fi - {% endif %} - - default_conf_dir=/opt/so/conf - - # Define a default directory to load pipelines from - ELASTICSEARCH_TEMPLATES="$default_conf_dir/elasticsearch/templates/" - - {% if GLOBALS.role == 'so-heavynode' %} - file="/opt/so/conf/elasticsearch/templates/index/so-common-template.json" - {% else %} - file="/usr/sbin/so-elastic-fleet-common" - {% endif %} - - if [ -f "$file" ]; then - # Wait for ElasticSearch to initialize - echo -n "Waiting for ElasticSearch..." - retry 240 1 "so-elasticsearch-query / -k --output /dev/null --silent --head --fail" || fail "Connection attempt timed out. Unable to connect to ElasticSearch. \nPlease try: \n -checking log(s) in /var/log/elasticsearch/\n -running 'sudo docker ps' \n -running 'sudo so-elastic-restart'" - {% if GLOBALS.role != 'so-heavynode' %} - TEMPLATE="logs-endpoint.alerts@package" - INSTALLED=$(so-elasticsearch-query _component_template/$TEMPLATE | jq -r .component_templates[0].name) - if [ "$INSTALLED" != "$TEMPLATE" ]; then - echo - echo "Packages not yet installed." - echo - exit 0 - fi - {% endif %} - - touch $STATE_FILE_INITIAL - - cd ${ELASTICSEARCH_TEMPLATES}/component/ecs - - echo "Loading ECS component templates..." - for i in *; do - TEMPLATE=$(echo $i | cut -d '.' -f1) - load_template "_component_template/${TEMPLATE}-mappings" "$i" - done - echo - - cd ${ELASTICSEARCH_TEMPLATES}/component/elastic-agent - - echo "Loading Elastic Agent component templates..." - {% if GLOBALS.role == 'so-heavynode' %} - component_pattern="so-*" - {% else %} - component_pattern="*" - {% endif %} - for i in $component_pattern; do - TEMPLATE=${i::-5} - load_template "_component_template/$TEMPLATE" "$i" - done - echo - - # Load SO-specific component templates - cd ${ELASTICSEARCH_TEMPLATES}/component/so - - echo "Loading Security Onion component templates..." - for i in *; do - TEMPLATE=$(echo $i | cut -d '.' -f1); - load_template "_component_template/$TEMPLATE" "$i" - done - echo - - # Load SO index templates - cd ${ELASTICSEARCH_TEMPLATES}/index - - echo "Loading Security Onion index templates..." - shopt -s extglob - {% if GLOBALS.role == 'so-heavynode' %} - pattern="!(*1password*|*aws*|*azure*|*cloudflare*|*elastic_agent*|*fim*|*github*|*google*|*osquery*|*system*|*windows*|*endpoint*|*elasticsearch*|*generic*|*fleet_server*|*soc*)" - {% else %} - pattern="*" - {% endif %} - # Index templates will be skipped if the following conditions are met: - # 1. The template is part of the "so-logs-" template group - # 2. The template name does not correlate to at least one existing component template - # In this situation, the script will treat the skipped template as a temporary failure - # and allow the templates to be loaded again on the next run or highstate, whichever - # comes first. - COMPONENT_LIST=$(so-elasticsearch-component-templates-list) - for i in $pattern; do - TEMPLATE=${i::-14} - COMPONENT_PATTERN=${TEMPLATE:3} - MATCH=$(echo "$TEMPLATE" | grep -E "^so-logs-|^so-metrics" | grep -vE "detections|osquery") - if [[ -n "$MATCH" && ! "$COMPONENT_LIST" =~ "$COMPONENT_PATTERN" && ! "$COMPONENT_PATTERN" =~ \.generic|logs-winlog\.winlog ]]; then - load_failures=$((load_failures+1)) - echo "Component template does not exist for $COMPONENT_PATTERN. The index template will not be loaded. Load failures: $load_failures" - else - load_template "_index_template/$TEMPLATE" "$i" - fi - done - else - {% if GLOBALS.role == 'so-heavynode' %} - echo "Common template does not exist. Exiting..." - {% else %} - echo "Elastic Fleet not configured. Exiting..." - {% endif %} - exit 0 - fi - - cd - >/dev/null - - if [[ $load_failures -eq 0 ]]; then - echo "All templates loaded successfully" - touch $STATE_FILE_SUCCESS - else - echo "Encountered $load_failures templates that were unable to load, likely due to missing dependencies that will be available later; will retry on next highstate" - fi -else - echo "Templates already loaded" -fi diff --git a/salt/firewall/containers.map.jinja b/salt/firewall/containers.map.jinja index 2d1135e5f..b39ba2b31 100644 --- a/salt/firewall/containers.map.jinja +++ b/salt/firewall/containers.map.jinja @@ -11,6 +11,7 @@ 'so-kratos', 'so-hydra', 'so-nginx', + 'so-postgres', 'so-redis', 'so-soc', 'so-strelka-coordinator', @@ -34,6 +35,7 @@ 'so-hydra', 'so-logstash', 'so-nginx', + 'so-postgres', 'so-redis', 'so-soc', 'so-strelka-coordinator', @@ -77,6 +79,7 @@ 'so-kratos', 'so-hydra', 'so-nginx', + 'so-postgres', 'so-soc' ] %} diff --git a/salt/firewall/defaults.yaml b/salt/firewall/defaults.yaml index a11492e88..5c1229787 100644 --- a/salt/firewall/defaults.yaml +++ b/salt/firewall/defaults.yaml @@ -98,6 +98,10 @@ firewall: tcp: - 8086 udp: [] + postgres: + tcp: + - 5432 + udp: [] kafka_controller: tcp: - 9093 @@ -193,6 +197,7 @@ firewall: - kibana - redis - influxdb + - postgres - elasticsearch_rest - elasticsearch_node - localrules @@ -379,6 +384,7 @@ firewall: - kibana - redis - influxdb + - postgres - elasticsearch_rest - elasticsearch_node - docker_registry @@ -392,6 +398,7 @@ firewall: - elasticsearch_rest - docker_registry - influxdb + - postgres - sensoroni - yum - beats_5044 @@ -404,6 +411,7 @@ firewall: portgroups: - docker_registry - influxdb + - postgres - sensoroni - yum - beats_5044 @@ -421,6 +429,7 @@ firewall: - yum - docker_registry - influxdb + - postgres - sensoroni searchnode: portgroups: @@ -431,6 +440,7 @@ firewall: - yum - docker_registry - influxdb + - postgres - elastic_agent_control - elastic_agent_data - elastic_agent_update @@ -444,6 +454,7 @@ firewall: - yum - docker_registry - influxdb + - postgres - elastic_agent_control - elastic_agent_data - elastic_agent_update @@ -453,6 +464,7 @@ firewall: - yum - docker_registry - influxdb + - postgres - elastic_agent_control - elastic_agent_data - elastic_agent_update @@ -486,6 +498,7 @@ firewall: portgroups: - docker_registry - influxdb + - postgres - sensoroni - yum - elastic_agent_control @@ -496,6 +509,7 @@ firewall: - yum - docker_registry - influxdb + - postgres - elastic_agent_control - elastic_agent_data - elastic_agent_update @@ -590,6 +604,7 @@ firewall: - kibana - redis - influxdb + - postgres - elasticsearch_rest - elasticsearch_node - docker_registry @@ -603,6 +618,7 @@ firewall: - elasticsearch_rest - docker_registry - influxdb + - postgres - sensoroni - yum - beats_5044 @@ -615,6 +631,7 @@ firewall: portgroups: - docker_registry - influxdb + - postgres - sensoroni - yum - beats_5044 @@ -632,6 +649,7 @@ firewall: - yum - docker_registry - influxdb + - postgres - sensoroni searchnode: portgroups: @@ -642,6 +660,7 @@ firewall: - yum - docker_registry - influxdb + - postgres - elastic_agent_control - elastic_agent_data - elastic_agent_update @@ -655,6 +674,7 @@ firewall: - yum - docker_registry - influxdb + - postgres - elastic_agent_control - elastic_agent_data - elastic_agent_update @@ -664,6 +684,7 @@ firewall: - yum - docker_registry - influxdb + - postgres - elastic_agent_control - elastic_agent_data - elastic_agent_update @@ -695,6 +716,7 @@ firewall: portgroups: - docker_registry - influxdb + - postgres - sensoroni - yum - elastic_agent_control @@ -705,6 +727,7 @@ firewall: - yum - docker_registry - influxdb + - postgres - elastic_agent_control - elastic_agent_data - elastic_agent_update @@ -799,6 +822,7 @@ firewall: - kibana - redis - influxdb + - postgres - elasticsearch_rest - elasticsearch_node - docker_registry @@ -812,6 +836,7 @@ firewall: - elasticsearch_rest - docker_registry - influxdb + - postgres - sensoroni - yum - beats_5044 @@ -824,6 +849,7 @@ firewall: portgroups: - docker_registry - influxdb + - postgres - sensoroni - yum - beats_5044 @@ -841,6 +867,7 @@ firewall: - yum - docker_registry - influxdb + - postgres - sensoroni searchnode: portgroups: @@ -850,6 +877,7 @@ firewall: - yum - docker_registry - influxdb + - postgres - elastic_agent_control - elastic_agent_data - elastic_agent_update @@ -862,6 +890,7 @@ firewall: - yum - docker_registry - influxdb + - postgres - elastic_agent_control - elastic_agent_data - elastic_agent_update @@ -871,6 +900,7 @@ firewall: - yum - docker_registry - influxdb + - postgres - elastic_agent_control - elastic_agent_data - elastic_agent_update @@ -904,6 +934,7 @@ firewall: portgroups: - docker_registry - influxdb + - postgres - sensoroni - yum - elastic_agent_control @@ -914,6 +945,7 @@ firewall: - yum - docker_registry - influxdb + - postgres - elastic_agent_control - elastic_agent_data - elastic_agent_update @@ -1011,6 +1043,7 @@ firewall: - kibana - redis - influxdb + - postgres - elasticsearch_rest - elasticsearch_node - docker_registry @@ -1031,6 +1064,7 @@ firewall: - elasticsearch_rest - docker_registry - influxdb + - postgres - sensoroni - yum - beats_5044 @@ -1043,6 +1077,7 @@ firewall: portgroups: - docker_registry - influxdb + - postgres - sensoroni - yum - beats_5044 @@ -1054,6 +1089,7 @@ firewall: portgroups: - docker_registry - influxdb + - postgres - sensoroni - yum - beats_5044 @@ -1065,6 +1101,7 @@ firewall: portgroups: - docker_registry - influxdb + - postgres - sensoroni - yum - redis @@ -1074,6 +1111,7 @@ firewall: portgroups: - docker_registry - influxdb + - postgres - sensoroni - yum - redis @@ -1084,6 +1122,7 @@ firewall: - yum - docker_registry - influxdb + - postgres - elastic_agent_control - elastic_agent_data - elastic_agent_update @@ -1120,6 +1159,7 @@ firewall: portgroups: - docker_registry - influxdb + - postgres - sensoroni - yum - elastic_agent_control @@ -1130,6 +1170,7 @@ firewall: - yum - docker_registry - influxdb + - postgres - elastic_agent_control - elastic_agent_data - elastic_agent_update @@ -1473,6 +1514,7 @@ firewall: - kibana - redis - influxdb + - postgres - elasticsearch_rest - elasticsearch_node - elastic_agent_control diff --git a/salt/global/soc_global.yaml b/salt/global/soc_global.yaml index 33abbf690..c15f3eb98 100644 --- a/salt/global/soc_global.yaml +++ b/salt/global/soc_global.yaml @@ -11,18 +11,14 @@ global: regexFailureMessage: You must enter a valid IP address or CIDR. mdengine: description: Which engine to use for meta data generation. Options are ZEEK and SURICATA. - regex: ^(ZEEK|SURICATA)$ options: - ZEEK - SURICATA - regexFailureMessage: You must enter either ZEEK or SURICATA. global: True pcapengine: description: Which engine to use for generating pcap. Currently only SURICATA is supported. - regex: ^(SURICATA)$ options: - SURICATA - regexFailureMessage: You must enter either SURICATA. global: True ids: description: Which IDS engine to use. Currently only Suricata is supported. @@ -42,11 +38,9 @@ global: advanced: True pipeline: description: Sets which pipeline technology for events to use. The use of Kafka requires a Security Onion Pro license. - regex: ^(REDIS|KAFKA)$ options: - REDIS - KAFKA - regexFailureMessage: You must enter either REDIS or KAFKA. global: True advanced: True repo_host: diff --git a/salt/influxdb/soc_influxdb.yaml b/salt/influxdb/soc_influxdb.yaml index 3dbf0875b..2b6bffe49 100644 --- a/salt/influxdb/soc_influxdb.yaml +++ b/salt/influxdb/soc_influxdb.yaml @@ -85,7 +85,10 @@ influxdb: description: The log level to use for outputting log statements. Allowed values are debug, info, or error. global: True advanced: false - regex: ^(info|debug|error)$ + options: + - info + - debug + - error helpLink: influxdb metrics-disabled: description: If true, the HTTP endpoint that exposes internal InfluxDB metrics will be inaccessible. @@ -140,7 +143,9 @@ influxdb: description: Determines the type of storage used for secrets. Allowed values are bolt or vault. global: True advanced: True - regex: ^(bolt|vault)$ + options: + - bolt + - vault helpLink: influxdb session-length: description: Number of minutes that a user login session can remain authenticated. @@ -260,7 +265,9 @@ influxdb: description: The type of data store to use for HTTP resources. Allowed values are disk or memory. Memory should not be used for production Security Onion installations. global: True advanced: True - regex: ^(disk|memory)$ + options: + - disk + - memory helpLink: influxdb tls-cert: description: The container path to the certificate to use for TLS encryption of the HTTP requests and responses. diff --git a/salt/kafka/soc_kafka.yaml b/salt/kafka/soc_kafka.yaml index b8d0c7c32..85469b8a4 100644 --- a/salt/kafka/soc_kafka.yaml +++ b/salt/kafka/soc_kafka.yaml @@ -128,10 +128,13 @@ kafka: title: ssl.keystore.password sensitive: True helpLink: kafka - ssl_x_keystore_x_type: + ssl_x_keystore_x_type: description: The key store file format. title: ssl.keystore.type - regex: ^(JKS|PKCS12|PEM)$ + options: + - JKS + - PKCS12 + - PEM helpLink: kafka ssl_x_truststore_x_location: description: The trust store file location within the Docker container. @@ -160,7 +163,11 @@ kafka: security_x_protocol: description: 'Broker communication protocol. Options are: SASL_SSL, PLAINTEXT, SSL, SASL_PLAINTEXT' title: security.protocol - regex: ^(SASL_SSL|PLAINTEXT|SSL|SASL_PLAINTEXT) + options: + - SASL_SSL + - PLAINTEXT + - SSL + - SASL_PLAINTEXT helpLink: kafka ssl_x_keystore_x_location: description: The key store file location within the Docker container. @@ -174,7 +181,10 @@ kafka: ssl_x_keystore_x_type: description: The key store file format. title: ssl.keystore.type - regex: ^(JKS|PKCS12|PEM)$ + options: + - JKS + - PKCS12 + - PEM helpLink: kafka ssl_x_truststore_x_location: description: The trust store file location within the Docker container. diff --git a/salt/kibana/defaults.yaml b/salt/kibana/defaults.yaml index 580891973..ecf56756b 100644 --- a/salt/kibana/defaults.yaml +++ b/salt/kibana/defaults.yaml @@ -22,7 +22,7 @@ kibana: - default - file migrations: - discardCorruptObjects: "8.18.8" + discardCorruptObjects: "9.3.3" telemetry: enabled: False xpack: diff --git a/salt/kibana/tools/sbin_jinja/so-kibana-space-defaults b/salt/kibana/tools/sbin_jinja/so-kibana-space-defaults index fcb80e606..d0447f514 100755 --- a/salt/kibana/tools/sbin_jinja/so-kibana-space-defaults +++ b/salt/kibana/tools/sbin_jinja/so-kibana-space-defaults @@ -9,5 +9,5 @@ SESSIONCOOKIE=$(curl -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http: # Disable certain Features from showing up in the Kibana UI echo echo "Setting up default Kibana Space:" -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["ml","enterpriseSearch","logs","infrastructure","apm","uptime","monitoring","stackAlerts","actions","securitySolutionCasesV3","inventory","dataQuality","searchSynonyms","enterpriseSearchApplications","enterpriseSearchAnalytics","securitySolutionTimeline","securitySolutionNotes","entityManager"]} ' >> /opt/so/log/kibana/misc.log +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["ml","enterpriseSearch","logs","infrastructure","apm","uptime","monitoring","stackAlerts","actions","securitySolutionCasesV3","inventory","dataQuality","searchSynonyms","searchQueryRules","enterpriseSearchApplications","enterpriseSearchAnalytics","securitySolutionTimeline","securitySolutionNotes","securitySolutionRulesV1","entityManager","streams","cloudConnect","slo"]} ' >> /opt/so/log/kibana/misc.log echo diff --git a/salt/kratos/soc_kratos.yaml b/salt/kratos/soc_kratos.yaml index 1cd2728c8..4cfe2c1c3 100644 --- a/salt/kratos/soc_kratos.yaml +++ b/salt/kratos/soc_kratos.yaml @@ -3,8 +3,8 @@ kratos: description: Enables or disables the Kratos authentication system. WARNING - Disabling this process will cause the grid to malfunction. Re-enabling this setting will require manual effort via SSH. forcedType: bool advanced: True + readonly: True helpLink: kratos - oidc: enabled: description: Set to True to enable OIDC / Single Sign-On (SSO) to SOC. Requires a valid Security Onion license key. @@ -21,8 +21,12 @@ kratos: description: "Specify the provider type. Required. Valid values are: auth0, generic, github, google, microsoft" global: True forcedType: string - regex: "auth0|generic|github|google|microsoft" - regexFailureMessage: "Valid values are: auth0, generic, github, google, microsoft" + options: + - auth0 + - generic + - github + - google + - microsoft helpLink: oidc client_id: description: Specify the client ID, also referenced as the application ID. Required. @@ -43,8 +47,9 @@ kratos: description: The source of the subject identifier. Typically 'userinfo'. Only used when provider is 'microsoft'. global: True forcedType: string - regex: me|userinfo - regexFailureMessage: "Valid values are: me, userinfo" + options: + - me + - userinfo helpLink: oidc auth_url: description: Provider's auth URL. Required when provider is 'generic'. diff --git a/salt/logstash/defaults.yaml b/salt/logstash/defaults.yaml index 520182555..db5e4ee58 100644 --- a/salt/logstash/defaults.yaml +++ b/salt/logstash/defaults.yaml @@ -26,12 +26,12 @@ logstash: manager: - so/0011_input_endgame.conf - so/0012_input_elastic_agent.conf.jinja - - so/0013_input_lumberjack_fleet.conf + - so/0013_input_lumberjack_fleet.conf.jinja - so/9999_output_redis.conf.jinja receiver: - so/0011_input_endgame.conf - so/0012_input_elastic_agent.conf.jinja - - so/0013_input_lumberjack_fleet.conf + - so/0013_input_lumberjack_fleet.conf.jinja - so/9999_output_redis.conf.jinja search: - so/0900_input_redis.conf.jinja @@ -69,4 +69,5 @@ logstash: pipeline_x_batch_x_size: 125 pipeline_x_ecs_compatibility: disabled dmz_nodes: [] + latency_metrics: False diff --git a/salt/logstash/pipelines/config/so/0012_input_elastic_agent.conf.jinja b/salt/logstash/pipelines/config/so/0012_input_elastic_agent.conf.jinja index a4d699aff..32dcac224 100644 --- a/salt/logstash/pipelines/config/so/0012_input_elastic_agent.conf.jinja +++ b/salt/logstash/pipelines/config/so/0012_input_elastic_agent.conf.jinja @@ -1,3 +1,4 @@ +{%- from 'logstash/map.jinja' import LOGSTASH_MERGED %} input { elastic_agent { port => 5055 @@ -11,10 +12,15 @@ input { } } filter { -if ![metadata] { - mutate { - rename => {"@metadata" => "metadata"} + {% if LOGSTASH_MERGED.get('latency_metrics', False) %} + ruby { + code => "event.set('[_tmp][logstash_from_agent]', Time.now().utc.iso8601(3));" + } + {% endif %} + if ![metadata] { + mutate { + rename => {"@metadata" => "metadata"} + } } } -} diff --git a/salt/logstash/pipelines/config/so/0013_input_lumberjack_fleet.conf b/salt/logstash/pipelines/config/so/0013_input_lumberjack_fleet.conf deleted file mode 100644 index b31ffee8d..000000000 --- a/salt/logstash/pipelines/config/so/0013_input_lumberjack_fleet.conf +++ /dev/null @@ -1,23 +0,0 @@ -input { - elastic_agent { - port => 5056 - tags => [ "elastic-agent", "fleet-lumberjack-input" ] - ssl_enabled => true - ssl_certificate => "/usr/share/logstash/elasticfleet-lumberjack.crt" - ssl_key => "/usr/share/logstash/elasticfleet-lumberjack.key" - ecs_compatibility => v8 - id => "fleet-lumberjack-in" - codec => "json" - } -} - - -filter { -if ![metadata] { - mutate { - rename => {"@metadata" => "metadata"} - } -} -} - - diff --git a/salt/logstash/pipelines/config/so/0013_input_lumberjack_fleet.conf.jinja b/salt/logstash/pipelines/config/so/0013_input_lumberjack_fleet.conf.jinja new file mode 100644 index 000000000..a04df5fd1 --- /dev/null +++ b/salt/logstash/pipelines/config/so/0013_input_lumberjack_fleet.conf.jinja @@ -0,0 +1,26 @@ +{%- from 'logstash/map.jinja' import LOGSTASH_MERGED %} +input { + elastic_agent { + port => 5056 + tags => [ "elastic-agent", "fleet-lumberjack-input" ] + ssl_enabled => true + ssl_certificate => "/usr/share/logstash/elasticfleet-lumberjack.crt" + ssl_key => "/usr/share/logstash/elasticfleet-lumberjack.key" + ecs_compatibility => v8 + id => "fleet-lumberjack-in" + codec => "json" + } +} + +filter { + {% if LOGSTASH_MERGED.get('latency_metrics', False) %} + ruby { + code => "event.set('[_tmp][logstash_from_fleet]', Time.now().utc.iso8601(3));" + } + {% endif %} + if ![metadata] { + mutate { + rename => {"@metadata" => "metadata"} + } + } +} diff --git a/salt/logstash/pipelines/config/so/0800_input_kafka.conf.jinja b/salt/logstash/pipelines/config/so/0800_input_kafka.conf.jinja index 7478375b0..769f71ea9 100644 --- a/salt/logstash/pipelines/config/so/0800_input_kafka.conf.jinja +++ b/salt/logstash/pipelines/config/so/0800_input_kafka.conf.jinja @@ -1,3 +1,4 @@ +{%- from 'logstash/map.jinja' import LOGSTASH_MERGED %} {%- set kafka_password = salt['pillar.get']('kafka:config:password') %} {%- set kafka_trustpass = salt['pillar.get']('kafka:config:trustpass') %} {%- set kafka_brokers = salt['pillar.get']('kafka:nodes', {}) %} @@ -30,6 +31,11 @@ input { } } filter { + {% if LOGSTASH_MERGED.get('latency_metrics', False) %} + ruby { + code => "event.set('[_tmp][logstash_from_kafka]', Time.now().utc.iso8601(3));" + } + {% endif %} if ![metadata] { mutate { rename => { "@metadata" => "metadata" } diff --git a/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja b/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja index ad9fae5f2..4bf388f4f 100644 --- a/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja +++ b/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja @@ -1,4 +1,4 @@ -{%- from 'logstash/map.jinja' import LOGSTASH_REDIS_NODES with context %} +{%- from 'logstash/map.jinja' import LOGSTASH_REDIS_NODES, LOGSTASH_MERGED %} {%- set REDIS_PASS = salt['pillar.get']('redis:config:requirepass') %} {%- for index in range(LOGSTASH_REDIS_NODES|length) %} @@ -18,3 +18,10 @@ input { } {% endfor %} {% endfor -%} +filter { + {% if LOGSTASH_MERGED.get('latency_metrics', False) %} + ruby { + code => "event.set('[_tmp][logstash_from_redis]', Time.now().utc.iso8601(3));" + } + {% endif %} +} diff --git a/salt/logstash/pipelines/config/so/9805_output_elastic_agent.conf.jinja b/salt/logstash/pipelines/config/so/9805_output_elastic_agent.conf.jinja index 4fe138dd8..f973070a5 100644 --- a/salt/logstash/pipelines/config/so/9805_output_elastic_agent.conf.jinja +++ b/salt/logstash/pipelines/config/so/9805_output_elastic_agent.conf.jinja @@ -1,3 +1,11 @@ +{%- from 'logstash/map.jinja' import LOGSTASH_MERGED %} +{% if LOGSTASH_MERGED.get('latency_metrics', False) %} +filter { + ruby { + code => "event.set('[_tmp][logstash_to_elasticsearch]', Time.now().utc.iso8601(3));" + } +} +{% endif %} output { if "elastic-agent" in [tags] and "so-ip-mappings" in [tags] { elasticsearch { diff --git a/salt/logstash/pipelines/config/so/9806_output_lumberjack_fleet.conf.jinja b/salt/logstash/pipelines/config/so/9806_output_lumberjack_fleet.conf.jinja index 50328e833..602c5fece 100644 --- a/salt/logstash/pipelines/config/so/9806_output_lumberjack_fleet.conf.jinja +++ b/salt/logstash/pipelines/config/so/9806_output_lumberjack_fleet.conf.jinja @@ -13,13 +13,20 @@ filter { add_tag => "fleet-lumberjack-{{ GLOBALS.hostname }}" } } - -output { - lumberjack { - codec => json +{%- from 'logstash/map.jinja' import LOGSTASH_MERGED %} +{% if LOGSTASH_MERGED.get('latency_metrics', False) %} +filter { + ruby { + code => "event.set('[_tmp][fleet_to_logstash]', Time.now().utc.iso8601(3));" + } +} +{% endif %} +output { + lumberjack { + codec => json hosts => {{ FAILOVER_LOGSTASH_NODES }} ssl_certificate => "/usr/share/filebeat/ca.crt" - port => 5056 + port => 5056 id => "fleet-lumberjack-{{ GLOBALS.hostname }}" - } + } } \ No newline at end of file diff --git a/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja b/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja index 0d3b3324b..af13915f7 100644 --- a/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja +++ b/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja @@ -1,10 +1,17 @@ +{%- from 'logstash/map.jinja' import LOGSTASH_MERGED %} {%- if grains.role in ['so-heavynode', 'so-receiver'] %} {%- set HOST = GLOBALS.hostname %} {%- else %} {%- set HOST = GLOBALS.manager %} {%- endif %} {%- set REDIS_PASS = salt['pillar.get']('redis:config:requirepass') %} - +{% if LOGSTASH_MERGED.get('latency_metrics', False) %} +filter { + ruby { + code => "event.set('[_tmp][logstash_to_redis]', Time.now().utc.iso8601(3));" + } +} +{% endif %} output { redis { host => '{{ HOST }}' diff --git a/salt/logstash/soc_logstash.yaml b/salt/logstash/soc_logstash.yaml index 5a5816a9e..40794afe4 100644 --- a/salt/logstash/soc_logstash.yaml +++ b/salt/logstash/soc_logstash.yaml @@ -86,3 +86,8 @@ logstash: multiline: True advanced: True forcedType: "[]string" + latency_metrics: + description: Enable latency metrics within events processed by logstash. Useful for pinpointing log ingest delay. + forcedType: bool + global: False + advanced: True diff --git a/salt/manager/tools/sbin/so-detections-overrides-import b/salt/manager/tools/sbin/so-detections-overrides-import new file mode 100755 index 000000000..e1cad3ac0 --- /dev/null +++ b/salt/manager/tools/sbin/so-detections-overrides-import @@ -0,0 +1,381 @@ +#!/usr/bin/env python3 + +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +# Imports detection overrides (e.g. from so-detections-backup) into the so-detection +# index. Reads . files (NDJSON, one override per line) from a source +# directory, looks up the matching detection by publicId+engine, validates each +# override against the same rules SOC enforces, dedupes against existing overrides +# (operational fields only), and appends new ones. + +import argparse +import ipaddress +import json +import os +import re +import sys +from datetime import datetime + +import requests +from requests.auth import HTTPBasicAuth +import urllib3 + +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + +DEFAULT_INDEX = "so-detection" +AUTH_FILE = "/opt/so/conf/elasticsearch/curl.config" +ES_URL = "https://localhost:9200" + +# Engines we know how to handle and the file extension the backup script writes. +ENGINES = { + "suricata": "txt", +} + +# Standard Suricata variables that ship with Security Onion. Anything else +# referenced in an override is "custom" and the user needs to make sure it +# exists in SOC Config before the override will function. +BUILTIN_SURICATA_VARS = { + "$HOME_NET", "$EXTERNAL_NET", + "$HTTP_SERVERS", "$DNS_SERVERS", "$SQL_SERVERS", "$SMTP_SERVERS", + "$TELNET_SERVERS", "$AIM_SERVERS", "$DC_SERVERS", "$MODBUS_SERVER", + "$MODBUS_CLIENT", "$ENIP_CLIENT", "$ENIP_SERVER", + "$HTTP_PORTS", "$SHELLCODE_PORTS", "$ORACLE_PORTS", "$SSH_PORTS", + "$FTP_PORTS", "$FILE_DATA_PORTS", +} + +VAR_PATTERN = re.compile(r"\$[A-Z_][A-Z0-9_]*") + +# Canonical valid values, per securityonion-soc/model/detection.go. +SURICATA_OVERRIDE_TYPES = {"suppress", "threshold", "modify"} +SUPPRESS_TRACKS = {"by_src", "by_dst", "by_either"} +THRESHOLD_TRACKS = {"by_src", "by_dst", "by_both"} +THRESHOLD_TYPES = {"limit", "threshold", "both"} + +STALE_WARNING = """\ +WARNING: so-detections-backup does not remove backup files when overrides are +deleted via the Security Onion web UI. As a result, files in the source +directory may represent overrides that were intentionally deleted and should +NOT be re-imported. + +Before continuing, verify that the source directory reflects the overrides you +actually want imported. Remove any files corresponding to overrides you previously deleted. +""" + + +def make_session(auth_file): + with open(auth_file, "r") as f: + for line in f: + if line.startswith("user ="): + creds = line.split("=", 1)[1].strip().replace('"', "") + user, _, password = creds.partition(":") + session = requests.Session() + session.auth = HTTPBasicAuth(user, password) + session.headers.update({"Content-Type": "application/json"}) + session.verify = False + return session + raise RuntimeError(f"Could not find 'user =' line in {auth_file}") + + +def find_detection(session, index, public_id, engine): + query = { + "query": {"bool": {"must": [ + {"term": {"so_detection.publicId": public_id}}, + {"term": {"so_detection.engine": engine}}, + ]}}, + "size": 2, + } + r = session.get(f"{ES_URL}/{index}/_search", json=query) + r.raise_for_status() + hits = r.json().get("hits", {}).get("hits", []) + if not hits: + return None, None, None + if len(hits) > 1: + # Shouldn't happen — publicId is unique per engine — but flag it. + print(f" WARN: {len(hits)} detections matched publicId={public_id} engine={engine}; using first") + hit = hits[0] + existing = hit["_source"].get("so_detection", {}).get("overrides") or [] + return hit["_id"], hit["_index"], existing + + +def update_overrides(session, doc_index, doc_id, overrides): + body = {"doc": {"so_detection": {"overrides": overrides}}} + r = session.post(f"{ES_URL}/{doc_index}/_update/{doc_id}", json=body) + r.raise_for_status() + return r.json() + + +def dedupe_key(override): + """Operational fields only, per Override.Equal() in detection.go. + Excludes timestamps and isEnabled so re-imports don't appear unique.""" + t = override.get("type") + if t == "suppress": + return (t, override.get("track"), override.get("ip")) + if t == "threshold": + return (t, override.get("thresholdType"), override.get("track"), + override.get("count"), override.get("seconds")) + if t == "modify": + return (t, override.get("regex"), override.get("value")) + + +def _validate_suricata_ip(ip): + if not ip: + return "ip cannot be empty" + if ip.startswith("$"): + return None + if ip.startswith("[") and ip.endswith("]"): + for part in ip[1:-1].split(","): + err = _validate_single_ip(part.strip()) + if err: + return f"invalid IP in list: {err}" + return None + return _validate_single_ip(ip) + + +def _validate_single_ip(ip): + try: + if "/" in ip: + ipaddress.ip_network(ip, strict=False) + else: + ipaddress.ip_address(ip) + except ValueError: + return f"invalid IP/CIDR {ip!r}" + return None + + +def validate_override(override, engine): + """Mirror Override.Validate() from securityonion-soc/model/detection.go. + Returns None on success, an error string otherwise.""" + t = override.get("type") + if not t: + return "override type is required" + if t not in SURICATA_OVERRIDE_TYPES: + return f"invalid type {t!r}: must be one of {sorted(SURICATA_OVERRIDE_TYPES)}" + + has = {k: override.get(k) is not None for k in + ("regex", "value", "thresholdType", "track", "ip", "count", "seconds", "customFilter")} + + if t == "suppress": + if not has["ip"] or not has["track"]: + return "suppress requires 'ip' and 'track'" + if any(has[k] for k in ("regex", "value", "thresholdType", "count", "seconds", "customFilter")): + return "suppress has unnecessary fields" + if override["track"] not in SUPPRESS_TRACKS: + return f"invalid track {override['track']!r}: must be one of {sorted(SUPPRESS_TRACKS)}" + return _validate_suricata_ip(override["ip"]) + + if t == "threshold": + if not all(has[k] for k in ("thresholdType", "track", "count", "seconds")): + return "threshold requires 'thresholdType', 'track', 'count', 'seconds'" + if any(has[k] for k in ("regex", "value", "customFilter")): + return "threshold has unnecessary fields" + if override["thresholdType"] not in THRESHOLD_TYPES: + return f"invalid thresholdType {override['thresholdType']!r}: must be one of {sorted(THRESHOLD_TYPES)}" + if override["track"] not in THRESHOLD_TRACKS: + return f"invalid track {override['track']!r}: must be one of {sorted(THRESHOLD_TRACKS)}" + if not isinstance(override["count"], int) or override["count"] <= 0: + return f"count must be a positive integer, got {override['count']!r}" + if not isinstance(override["seconds"], int) or override["seconds"] <= 0: + return f"seconds must be a positive integer, got {override['seconds']!r}" + return None + + if t == "modify": + if not has["regex"] or not has["value"]: + return "modify requires 'regex' and 'value'" + if any(has[k] for k in ("thresholdType", "track", "count", "seconds", "customFilter")): + return "modify has unnecessary fields" + try: + re.compile(override["regex"]) + except re.error as e: + return f"invalid regex: {e}" + return None + + +def parse_overrides_file(path): + """Parse a file written by so-detections-backup.py: NDJSON, one override + per line. Returns a list of (override_dict, line_number).""" + overrides = [] + with open(path, "r") as f: + for i, line in enumerate(f, start=1): + line = line.strip() + if not line: + continue + overrides.append((json.loads(line), i)) + return overrides + + +def describe(override): + """Human-readable summary of the operational fields for a given override type.""" + t = override.get("type") + if t == "suppress": + return f"type=suppress track={override.get('track')} ip={override.get('ip')}" + if t == "threshold": + return (f"type=threshold track={override.get('track')} " + f"thresholdType={override.get('thresholdType')} " + f"count={override.get('count')} seconds={override.get('seconds')}") + if t == "modify": + return f"type=modify regex={override.get('regex')!r}" + + +def collect_custom_vars(override): + found = set() + for value in override.values(): + if isinstance(value, str): + for match in VAR_PATTERN.findall(value): + if match not in BUILTIN_SURICATA_VARS: + found.add(match) + return found + + +def parse_args(): + p = argparse.ArgumentParser( + description="Import detection overrides into the so-detection index.", + ) + p.add_argument("--source", "-s", required=True, + help="Source directory containing . override files.") + p.add_argument("--engine", "-e", default="suricata", choices=list(ENGINES.keys()), + help="Detection engine (default: suricata).") + p.add_argument("--dry-run", "-n", action="store_true", + help="Print what would happen without writing to Elasticsearch.") + p.add_argument("--no-import-note", action="store_true", + help="Do not prepend '[Imported YYYY-MM-DD] ' to the override note.") + p.add_argument("--index", "-i", default=DEFAULT_INDEX, + help=f"Elasticsearch index to update (default: {DEFAULT_INDEX}).") + return p.parse_args() + + +def confirm_proceed(args): + """Show the stale-backup warning. Dry-run prints it and continues. Real + runs require the user typing 'yes' at the prompt.""" + print(STALE_WARNING) + if args.dry_run: + print("(dry-run: no acknowledgement required)\n") + return True + answer = input("Type 'yes' to acknowledge and continue: ").strip().lower() + print() + return answer == "yes" + + +def main(): + args = parse_args() + + if not os.path.isdir(args.source): + print(f"ERROR: source directory not found: {args.source}", file=sys.stderr) + sys.exit(1) + + extension = ENGINES[args.engine] + files = sorted(f for f in os.listdir(args.source) if f.endswith(f".{extension}")) + if not files: + print(f"No *.{extension} files found in {args.source}") + sys.exit(0) + + if not confirm_proceed(args): + print("Aborted.") + sys.exit(1) + + session = make_session(AUTH_FILE) + today = datetime.now().strftime("%Y-%m-%d") + note_prefix = "" if args.no_import_note else f"[Imported {today}] " + + counts = {"added": 0, "skipped_dedupe": 0, "skipped_not_found": 0, "invalid": 0, "error": 0} + custom_vars = set() + + mode = "DRY-RUN" if args.dry_run else "IMPORT" + print(f"[{mode}] engine={args.engine} source={args.source} index={args.index}\n") + + for filename in files: + public_id = os.path.splitext(filename)[0] + path = os.path.join(args.source, filename) + print(f"{public_id}:") + + try: + new_overrides = parse_overrides_file(path) + except (json.JSONDecodeError, OSError) as e: + print(f" ERROR: could not parse {filename}: {e}") + counts["error"] += 1 + continue + + if not new_overrides: + print(" SKIP: empty file") + continue + + try: + doc_id, doc_index, existing = find_detection(session, args.index, public_id, args.engine) + except requests.HTTPError as e: + print(f" ERROR: search failed: {e}") + counts["error"] += 1 + continue + + if doc_id is None: + print(f" WARN: no detection found for publicId={public_id} engine={args.engine}; skipping") + counts["skipped_not_found"] += len(new_overrides) + continue + + existing_keys = {dedupe_key(o) for o in existing} + merged = list(existing) + added_this_file = 0 + + for override, line_no in new_overrides: + err = validate_override(override, args.engine) + if err: + print(f" INVALID (line {line_no}): {err}") + counts["invalid"] += 1 + continue + + custom_vars.update(collect_custom_vars(override)) + key = dedupe_key(override) + if key in existing_keys: + print(f" SKIP (line {line_no}): duplicate of existing override [{describe(override)}]") + counts["skipped_dedupe"] += 1 + continue + + if note_prefix: + override = dict(override) + override["note"] = note_prefix + (override.get("note") or "") + + merged.append(override) + existing_keys.add(key) + added_this_file += 1 + print(f" ADD (line {line_no}): {describe(override)}") + + if added_this_file == 0: + continue + + if args.dry_run: + print(f" DRY-RUN: would update {doc_index}/{doc_id} " + f"({len(existing)} existing → {len(merged)} total)") + counts["added"] += added_this_file + continue + + try: + update_overrides(session, doc_index, doc_id, merged) + print(f" UPDATED {doc_index}/{doc_id} ({len(existing)} → {len(merged)})") + counts["added"] += added_this_file + except requests.HTTPError as e: + print(f" ERROR: update failed: {e}") + counts["error"] += 1 + + print() + print("=" * 60) + print(f"Summary ({mode}):") + print(f" Overrides added: {counts['added']}") + print(f" Skipped (already present): {counts['skipped_dedupe']}") + print(f" Skipped (no detection): {counts['skipped_not_found']}") + print(f" Invalid (failed checks): {counts['invalid']}") + print(f" Errors: {counts['error']}") + + if custom_vars: + print() + print("WARNING: detected custom Suricata variables in imported overrides:") + for v in sorted(custom_vars): + print(f" {v}") + print("If any of these are not already defined in SOC Config (Suricata variables),") + print("you must add them manually before the rules will function correctly.") + + sys.exit(0 if counts["error"] == 0 and counts["invalid"] == 0 else 1) + + +if __name__ == "__main__": + main() diff --git a/salt/manager/tools/sbin/so-detections-overrides-import_test.py b/salt/manager/tools/sbin/so-detections-overrides-import_test.py new file mode 100644 index 000000000..5f5361ea4 --- /dev/null +++ b/salt/manager/tools/sbin/so-detections-overrides-import_test.py @@ -0,0 +1,588 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +import importlib.util +import json +import os +import shutil +import sys +import tempfile +import unittest +from importlib.machinery import SourceFileLoader +from io import StringIO +from unittest.mock import MagicMock, patch + +import requests + +# The script has no .py extension; spec_from_file_location can't auto-detect a +# loader, so we hand it a SourceFileLoader explicitly. (load_module() is +# deprecated in 3.14 and slated for removal in 3.15.) +HERE = os.path.dirname(os.path.abspath(__file__)) +SCRIPT = os.path.join(HERE, "so-detections-overrides-import") +_loader = SourceFileLoader("so_overrides_import", SCRIPT) +_spec = importlib.util.spec_from_loader("so_overrides_import", _loader) +soi = importlib.util.module_from_spec(_spec) +_loader.exec_module(soi) + + +class TestValidateSuppress(unittest.TestCase): + def test_valid(self): + self.assertIsNone(soi.validate_override( + {"type": "suppress", "track": "by_src", "ip": "1.2.3.4"}, "suricata")) + + def test_valid_var(self): + self.assertIsNone(soi.validate_override( + {"type": "suppress", "track": "by_either", "ip": "$HOME_NET"}, "suricata")) + + def test_valid_cidr(self): + self.assertIsNone(soi.validate_override( + {"type": "suppress", "track": "by_dst", "ip": "10.0.0.0/8"}, "suricata")) + + def test_valid_bracket_list(self): + self.assertIsNone(soi.validate_override( + {"type": "suppress", "track": "by_src", "ip": "[1.2.3.4,10.0.0.0/8]"}, "suricata")) + + def test_missing_ip(self): + err = soi.validate_override({"type": "suppress", "track": "by_src"}, "suricata") + self.assertIn("requires", err) + + def test_missing_track(self): + err = soi.validate_override({"type": "suppress", "ip": "1.2.3.4"}, "suricata") + self.assertIn("requires", err) + + def test_invalid_track(self): + err = soi.validate_override( + {"type": "suppress", "track": "by_both", "ip": "1.2.3.4"}, "suricata") + self.assertIn("invalid track", err) + + def test_invalid_ip(self): + err = soi.validate_override( + {"type": "suppress", "track": "by_src", "ip": "not-an-ip"}, "suricata") + self.assertIn("invalid IP", err) + + def test_unnecessary_field(self): + err = soi.validate_override( + {"type": "suppress", "track": "by_src", "ip": "1.2.3.4", "count": 5}, "suricata") + self.assertIn("unnecessary fields", err) + + +class TestValidateThreshold(unittest.TestCase): + def test_valid(self): + self.assertIsNone(soi.validate_override({ + "type": "threshold", "track": "by_src", + "thresholdType": "limit", "count": 10, "seconds": 60, + }, "suricata")) + + def test_valid_by_both(self): + self.assertIsNone(soi.validate_override({ + "type": "threshold", "track": "by_both", + "thresholdType": "both", "count": 1, "seconds": 1, + }, "suricata")) + + def test_track_by_either_invalid(self): + err = soi.validate_override({ + "type": "threshold", "track": "by_either", + "thresholdType": "limit", "count": 10, "seconds": 60, + }, "suricata") + self.assertIn("invalid track", err) + + def test_invalid_threshold_type(self): + err = soi.validate_override({ + "type": "threshold", "track": "by_src", + "thresholdType": "bogus", "count": 10, "seconds": 60, + }, "suricata") + self.assertIn("invalid thresholdType", err) + + def test_zero_count(self): + err = soi.validate_override({ + "type": "threshold", "track": "by_src", + "thresholdType": "limit", "count": 0, "seconds": 60, + }, "suricata") + self.assertIn("count", err) + + def test_negative_seconds(self): + err = soi.validate_override({ + "type": "threshold", "track": "by_src", + "thresholdType": "limit", "count": 10, "seconds": -1, + }, "suricata") + self.assertIn("seconds", err) + + def test_missing_field(self): + err = soi.validate_override({ + "type": "threshold", "track": "by_src", + "thresholdType": "limit", "count": 10, # missing seconds + }, "suricata") + self.assertIn("requires", err) + + def test_unnecessary_field(self): + err = soi.validate_override({ + "type": "threshold", "track": "by_src", + "thresholdType": "limit", "count": 10, "seconds": 60, + "regex": "foo", + }, "suricata") + self.assertIn("unnecessary fields", err) + + +class TestValidateModify(unittest.TestCase): + def test_valid(self): + self.assertIsNone(soi.validate_override( + {"type": "modify", "regex": r"content:\"foo\"", "value": "content:bar"}, "suricata")) + + def test_invalid_regex(self): + err = soi.validate_override( + {"type": "modify", "regex": "(unbalanced", "value": "x"}, "suricata") + self.assertIn("invalid regex", err) + + def test_missing_value(self): + err = soi.validate_override({"type": "modify", "regex": "x"}, "suricata") + self.assertIn("requires", err) + + def test_unnecessary_field(self): + err = soi.validate_override( + {"type": "modify", "regex": "x", "value": "y", "track": "by_src"}, "suricata") + self.assertIn("unnecessary fields", err) + + +class TestValidateMisc(unittest.TestCase): + def test_unknown_type(self): + err = soi.validate_override({"type": "suppresss", "track": "by_src", "ip": "1.2.3.4"}, "suricata") + self.assertIn("invalid type", err) + + def test_missing_type(self): + err = soi.validate_override({"track": "by_src"}, "suricata") + self.assertIn("type is required", err) + + +class TestValidateIP(unittest.TestCase): + def test_plain_ipv4(self): + self.assertIsNone(soi._validate_suricata_ip("1.2.3.4")) + + def test_plain_ipv6(self): + self.assertIsNone(soi._validate_suricata_ip("::1")) + + def test_cidr(self): + self.assertIsNone(soi._validate_suricata_ip("10.0.0.0/8")) + + def test_var(self): + self.assertIsNone(soi._validate_suricata_ip("$CONCOURSEWORKERS")) + + def test_bracket_list(self): + self.assertIsNone(soi._validate_suricata_ip("[1.2.3.4, 10.0.0.0/8]")) + + def test_bracket_list_bad_member(self): + err = soi._validate_suricata_ip("[1.2.3.4,nope]") + self.assertIn("invalid IP in list", err) + + def test_empty(self): + self.assertIn("empty", soi._validate_suricata_ip("")) + + def test_invalid(self): + self.assertIn("invalid", soi._validate_suricata_ip("999.999.999.999")) + + +class TestDedupeKey(unittest.TestCase): + def test_suppress(self): + a = {"type": "suppress", "track": "by_src", "ip": "1.2.3.4", "count": 99} + b = {"type": "suppress", "track": "by_src", "ip": "1.2.3.4"} + # count is irrelevant for suppress dedupe + self.assertEqual(soi.dedupe_key(a), soi.dedupe_key(b)) + + def test_suppress_differs_on_ip(self): + a = {"type": "suppress", "track": "by_src", "ip": "1.2.3.4"} + b = {"type": "suppress", "track": "by_src", "ip": "5.6.7.8"} + self.assertNotEqual(soi.dedupe_key(a), soi.dedupe_key(b)) + + def test_threshold(self): + a = {"type": "threshold", "track": "by_src", "thresholdType": "limit", + "count": 10, "seconds": 60, "ip": "ignored"} + b = {"type": "threshold", "track": "by_src", "thresholdType": "limit", + "count": 10, "seconds": 60} + self.assertEqual(soi.dedupe_key(a), soi.dedupe_key(b)) + + def test_threshold_differs_on_count(self): + a = {"type": "threshold", "track": "by_src", "thresholdType": "limit", + "count": 10, "seconds": 60} + b = {"type": "threshold", "track": "by_src", "thresholdType": "limit", + "count": 20, "seconds": 60} + self.assertNotEqual(soi.dedupe_key(a), soi.dedupe_key(b)) + + def test_modify(self): + a = {"type": "modify", "regex": "x", "value": "y"} + b = {"type": "modify", "regex": "x", "value": "y"} + self.assertEqual(soi.dedupe_key(a), soi.dedupe_key(b)) + + +class TestDescribe(unittest.TestCase): + def test_suppress(self): + s = soi.describe({"type": "suppress", "track": "by_src", "ip": "1.2.3.4"}) + self.assertIn("suppress", s) + self.assertIn("by_src", s) + self.assertIn("1.2.3.4", s) + + def test_threshold_includes_count(self): + s = soi.describe({"type": "threshold", "track": "by_src", + "thresholdType": "limit", "count": 10, "seconds": 60}) + self.assertIn("count=10", s) + self.assertIn("seconds=60", s) + + def test_modify(self): + s = soi.describe({"type": "modify", "regex": "foo"}) + self.assertIn("modify", s) + self.assertIn("foo", s) + + +class TestParseOverridesFile(unittest.TestCase): + def _write(self, content): + fd, path = tempfile.mkstemp(suffix=".txt") + os.close(fd) + with open(path, "w") as f: + f.write(content) + self.addCleanup(os.unlink, path) + return path + + def test_single_line(self): + path = self._write('{"type":"suppress","track":"by_src","ip":"1.2.3.4"}') + result = soi.parse_overrides_file(path) + self.assertEqual(len(result), 1) + self.assertEqual(result[0][0]["type"], "suppress") + self.assertEqual(result[0][1], 1) + + def test_ndjson(self): + path = self._write( + '{"type":"suppress","track":"by_src","ip":"1.2.3.4"}\n' + '{"type":"suppress","track":"by_dst","ip":"5.6.7.8"}\n' + ) + result = soi.parse_overrides_file(path) + self.assertEqual(len(result), 2) + self.assertEqual(result[1][1], 2) + + def test_empty(self): + path = self._write("") + self.assertEqual(soi.parse_overrides_file(path), []) + + def test_blank_lines_skipped(self): + path = self._write('\n{"type":"suppress","track":"by_src","ip":"1.2.3.4"}\n\n') + result = soi.parse_overrides_file(path) + self.assertEqual(len(result), 1) + self.assertEqual(result[0][1], 2) # line number reflects original position + + def test_invalid_raises(self): + path = self._write("not json") + with self.assertRaises(json.JSONDecodeError): + soi.parse_overrides_file(path) + + +class TestCollectCustomVars(unittest.TestCase): + def test_finds_custom(self): + v = soi.collect_custom_vars({"ip": "$CONCOURSEWORKERS"}) + self.assertEqual(v, {"$CONCOURSEWORKERS"}) + + def test_filters_builtins(self): + v = soi.collect_custom_vars({"ip": "$HOME_NET"}) + self.assertEqual(v, set()) + + def test_mixed(self): + v = soi.collect_custom_vars({"ip": "[$HOME_NET,$MYNET]"}) + self.assertEqual(v, {"$MYNET"}) + + def test_non_string_fields_ignored(self): + v = soi.collect_custom_vars({"count": 10, "isEnabled": True}) + self.assertEqual(v, set()) + + +class TestMakeSession(unittest.TestCase): + def _write(self, content): + fd, path = tempfile.mkstemp() + os.close(fd) + with open(path, "w") as f: + f.write(content) + self.addCleanup(os.unlink, path) + return path + + def test_valid_auth_file(self): + path = self._write('user = "admin:secret"\n') + session = soi.make_session(path) + self.assertEqual(session.auth.username, "admin") + self.assertEqual(session.auth.password, "secret") + self.assertFalse(session.verify) + + def test_missing_user_line(self): + path = self._write("# no user line here\n") + with self.assertRaises(RuntimeError): + soi.make_session(path) + + +class TestFindDetection(unittest.TestCase): + def _session_with_response(self, payload): + session = MagicMock() + response = MagicMock() + response.json.return_value = payload + response.raise_for_status.return_value = None + session.get.return_value = response + return session + + def test_found(self): + session = self._session_with_response({"hits": {"hits": [{ + "_id": "abc", "_index": "so-detection", + "_source": {"so_detection": {"overrides": [{"type": "suppress"}]}}, + }]}}) + doc_id, idx, existing = soi.find_detection(session, "so-detection", "2049201", "suricata") + self.assertEqual(doc_id, "abc") + self.assertEqual(idx, "so-detection") + self.assertEqual(len(existing), 1) + + def test_not_found(self): + session = self._session_with_response({"hits": {"hits": []}}) + doc_id, idx, existing = soi.find_detection(session, "so-detection", "x", "suricata") + self.assertIsNone(doc_id) + self.assertIsNone(idx) + self.assertIsNone(existing) + + def test_no_overrides_field(self): + session = self._session_with_response({"hits": {"hits": [{ + "_id": "abc", "_index": "so-detection", + "_source": {"so_detection": {}}, + }]}}) + _, _, existing = soi.find_detection(session, "so-detection", "x", "suricata") + self.assertEqual(existing, []) + + def test_multiple_hits_warns(self): + session = self._session_with_response({"hits": {"hits": [ + {"_id": "a", "_index": "i", "_source": {"so_detection": {"overrides": []}}}, + {"_id": "b", "_index": "i", "_source": {"so_detection": {"overrides": []}}}, + ]}}) + with patch("sys.stdout", new=StringIO()) as out: + doc_id, _, _ = soi.find_detection(session, "i", "x", "suricata") + self.assertEqual(doc_id, "a") + self.assertIn("WARN", out.getvalue()) + + +class TestUpdateOverrides(unittest.TestCase): + def test_posts_to_update_endpoint(self): + session = MagicMock() + response = MagicMock() + response.raise_for_status.return_value = None + response.json.return_value = {"result": "updated"} + session.post.return_value = response + + result = soi.update_overrides(session, "so-detection", "abc", [{"type": "suppress"}]) + + self.assertEqual(result, {"result": "updated"}) + url = session.post.call_args[0][0] + self.assertIn("/_update/abc", url) + body = session.post.call_args[1]["json"] + self.assertEqual(body["doc"]["so_detection"]["overrides"], [{"type": "suppress"}]) + + +class TestConfirmProceed(unittest.TestCase): + def test_dry_run_skips_prompt(self): + args = MagicMock(dry_run=True) + with patch("sys.stdout", new=StringIO()): + self.assertTrue(soi.confirm_proceed(args)) + + def test_yes_input(self): + args = MagicMock(dry_run=False) + with patch("sys.stdout", new=StringIO()): + with patch("builtins.input", return_value="yes"): + self.assertTrue(soi.confirm_proceed(args)) + + def test_yes_input_case_insensitive(self): + args = MagicMock(dry_run=False) + with patch("sys.stdout", new=StringIO()): + with patch("builtins.input", return_value="YES"): + self.assertTrue(soi.confirm_proceed(args)) + + def test_no_input_aborts(self): + args = MagicMock(dry_run=False) + with patch("sys.stdout", new=StringIO()): + with patch("builtins.input", return_value="no"): + self.assertFalse(soi.confirm_proceed(args)) + + def test_empty_input_aborts(self): + args = MagicMock(dry_run=False) + with patch("sys.stdout", new=StringIO()): + with patch("builtins.input", return_value=""): + self.assertFalse(soi.confirm_proceed(args)) + + +class TestParseArgs(unittest.TestCase): + def test_defaults(self): + with patch.object(sys, "argv", ["cmd", "--source", "/some/path"]): + args = soi.parse_args() + self.assertEqual(args.source, "/some/path") + self.assertEqual(args.engine, "suricata") + self.assertFalse(args.dry_run) + self.assertFalse(args.no_import_note) + self.assertEqual(args.index, soi.DEFAULT_INDEX) + + def test_all_options(self): + argv = ["cmd", "-s", "/x", "-e", "suricata", "-n", + "--no-import-note", "-i", "alt-index"] + with patch.object(sys, "argv", argv): + args = soi.parse_args() + self.assertEqual(args.source, "/x") + self.assertTrue(args.dry_run) + self.assertTrue(args.no_import_note) + self.assertEqual(args.index, "alt-index") + + +class TestMain(unittest.TestCase): + def setUp(self): + self.tmpdir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmpdir, ignore_errors=True) + # Stub make_session so tests don't need /opt/so/conf/elasticsearch/curl.config. + p = patch.object(soi, "make_session", return_value=MagicMock()) + p.start() + self.addCleanup(p.stop) + + def _write_file(self, public_id, overrides, ext="txt"): + """Write an NDJSON override file. Entries may be dicts or raw strings (for malformed input).""" + path = os.path.join(self.tmpdir, f"{public_id}.{ext}") + with open(path, "w") as f: + for o in overrides: + f.write(o if isinstance(o, str) else json.dumps(o)) + f.write("\n") + return path + + def _run_main(self, *extra_argv, input_response="yes"): + """Run main() with stdout/stderr captured and input mocked. Returns (stdout, stderr, exit_code).""" + argv = ["cmd", "--source", self.tmpdir, *extra_argv] + out, err = StringIO(), StringIO() + with patch.object(sys, "argv", argv), \ + patch("sys.stdout", new=out), \ + patch("sys.stderr", new=err), \ + patch("builtins.input", return_value=input_response): + with self.assertRaises(SystemExit) as cm: + soi.main() + return out.getvalue(), err.getvalue(), cm.exception.code + + def test_source_dir_missing(self): + argv = ["cmd", "--source", "/no/such/path/here"] + err = StringIO() + with patch.object(sys, "argv", argv), patch("sys.stderr", new=err): + with self.assertRaises(SystemExit) as cm: + soi.main() + self.assertEqual(cm.exception.code, 1) + self.assertIn("source directory not found", err.getvalue()) + + def test_no_files_found(self): + out, _, code = self._run_main() + self.assertEqual(code, 0) + self.assertIn("No *.txt files found", out) + + def test_user_aborts(self): + self._write_file("1001", [{"type": "suppress", "track": "by_src", "ip": "1.2.3.4"}]) + out, _, code = self._run_main(input_response="no") + self.assertEqual(code, 1) + self.assertIn("Aborted", out) + + def test_parse_error_increments_error(self): + # Malformed JSON line — parse_overrides_file raises JSONDecodeError. + self._write_file("1002", ["not json"]) + out, _, code = self._run_main("--dry-run") + self.assertEqual(code, 1) # invalid+error → non-zero + self.assertIn("could not parse", out) + self.assertIn("Errors: 1", out) + + def test_empty_file_skipped(self): + # Blank lines only — parse_overrides_file returns []; main reports "empty file" and continues. + path = os.path.join(self.tmpdir, "1003.txt") + with open(path, "w") as f: + f.write("\n\n") + out, _, code = self._run_main("--dry-run") + self.assertEqual(code, 0) + self.assertIn("empty file", out) + + @patch.object(soi, "find_detection") + def test_search_http_error(self, mock_find): + mock_find.side_effect = requests.HTTPError("boom") + self._write_file("1004", [{"type": "suppress", "track": "by_src", "ip": "1.2.3.4"}]) + out, _, code = self._run_main("--dry-run") + self.assertEqual(code, 1) + self.assertIn("search failed", out) + + @patch.object(soi, "find_detection") + def test_no_detection_found(self, mock_find): + mock_find.return_value = (None, None, None) + self._write_file("1005", [{"type": "suppress", "track": "by_src", "ip": "1.2.3.4"}]) + out, _, code = self._run_main("--dry-run") + self.assertEqual(code, 0) + self.assertIn("no detection found", out) + self.assertIn("Skipped (no detection): 1", out) + + @patch.object(soi, "find_detection") + def test_all_duplicates_no_update(self, mock_find): + existing = [{"type": "suppress", "track": "by_src", "ip": "1.2.3.4"}] + mock_find.return_value = ("doc1", "so-detection", existing) + self._write_file("1006", [{"type": "suppress", "track": "by_src", "ip": "1.2.3.4"}]) + out, _, code = self._run_main("--dry-run") + self.assertEqual(code, 0) + self.assertIn("SKIP", out) + self.assertNotIn("DRY-RUN: would update", out) # added_this_file == 0 branch + + @patch.object(soi, "update_overrides") + @patch.object(soi, "find_detection") + def test_happy_path_full(self, mock_find, mock_update): + # Exercises: ADD, dedupe SKIP, INVALID, note prefix, UPDATE, custom-vars warning, exit=1 (invalid present) + existing = [{"type": "suppress", "track": "by_src", "ip": "9.9.9.9"}] + mock_find.return_value = ("doc1", "so-detection", existing) + mock_update.return_value = {"result": "updated"} + self._write_file("1007", [ + {"type": "suppress", "track": "by_src", "ip": "1.2.3.4"}, # ADD + {"type": "suppress", "track": "by_src", "ip": "9.9.9.9"}, # SKIP (dupe of existing) + {"type": "suppress", "track": "bogus", "ip": "1.2.3.4"}, # INVALID + {"type": "suppress", "track": "by_src", "ip": "$CONCOURSEWORKERS"}, # ADD + custom var + ]) + out, _, code = self._run_main() + self.assertEqual(code, 1) # one invalid -> non-zero + + mock_update.assert_called_once() + merged = mock_update.call_args[0][3] + self.assertEqual(len(merged), 3) # 1 existing + 2 new + new_notes = [o.get("note", "") for o in merged if o.get("ip") in ("1.2.3.4", "$CONCOURSEWORKERS")] + self.assertTrue(all(n.startswith("[Imported ") for n in new_notes)) + + self.assertIn("ADD", out) + self.assertIn("SKIP", out) + self.assertIn("INVALID", out) + self.assertIn("UPDATED", out) + self.assertIn("$CONCOURSEWORKERS", out) + + @patch.object(soi, "update_overrides") + @patch.object(soi, "find_detection") + def test_no_import_note_preserves_note(self, mock_find, mock_update): + mock_find.return_value = ("doc1", "so-detection", []) + mock_update.return_value = {"result": "updated"} + self._write_file("1008", [ + {"type": "suppress", "track": "by_src", "ip": "1.2.3.4", "note": "original"}, + ]) + _, _, code = self._run_main("--no-import-note") + self.assertEqual(code, 0) + merged = mock_update.call_args[0][3] + self.assertEqual(merged[0]["note"], "original") # no prefix applied + + @patch.object(soi, "find_detection") + def test_dry_run_skips_update(self, mock_find): + mock_find.return_value = ("doc1", "so-detection", []) + self._write_file("1009", [{"type": "suppress", "track": "by_src", "ip": "1.2.3.4"}]) + with patch.object(soi, "update_overrides") as mock_update: + out, _, code = self._run_main("--dry-run") + self.assertEqual(code, 0) + mock_update.assert_not_called() + self.assertIn("DRY-RUN: would update", out) + + @patch.object(soi, "update_overrides") + @patch.object(soi, "find_detection") + def test_update_http_error(self, mock_find, mock_update): + mock_find.return_value = ("doc1", "so-detection", []) + mock_update.side_effect = requests.HTTPError("nope") + self._write_file("1010", [{"type": "suppress", "track": "by_src", "ip": "1.2.3.4"}]) + out, _, code = self._run_main() + self.assertEqual(code, 1) + self.assertIn("update failed", out) + + +if __name__ == "__main__": + unittest.main() diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index 2d5ef448e..86bab25e6 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -132,8 +132,8 @@ function getinstallinfo() { log "ERROR" "Failed to get install info from $MINION_ID" return 1 fi - - export $(echo "$INSTALLVARS" | xargs) + + while read -r var; do export "$var"; done <<< "$INSTALLVARS" if [ $? -ne 0 ]; then log "ERROR" "Failed to source install variables" return 1 @@ -273,7 +273,7 @@ function deleteMinionFiles () { log "ERROR" "Failed to delete $PILLARFILE" return 1 fi - + rm -f $ADVPILLARFILE if [ $? -ne 0 ]; then log "ERROR" "Failed to delete $ADVPILLARFILE" @@ -281,6 +281,39 @@ function deleteMinionFiles () { fi } +# Remove this minion's postgres Telegraf credential from the shared creds +# pillar and drop the matching role in Postgres. Always returns 0 so a dead +# or unreachable so-postgres doesn't block minion deletion — in that case we +# log a warning and leave the role behind for manual cleanup. +function remove_postgres_telegraf_from_minion() { + local MINION_SAFE + MINION_SAFE=$(echo "$MINION_ID" | tr '.-' '__' | tr '[:upper:]' '[:lower:]') + local PG_USER="so_telegraf_${MINION_SAFE}" + + log "INFO" "Removing postgres telegraf cred for $MINION_ID" + + so-telegraf-cred remove "$MINION_ID" >/dev/null 2>&1 || true + + if docker ps --format '{{.Names}}' 2>/dev/null | grep -q '^so-postgres$'; then + if ! docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -U postgres -d so_telegraf >/dev/null 2>&1 < " >&2 + exit 2 +} + +seed_creds_file() { + mkdir -p "$(dirname "$CREDS")" || return 1 + if [[ ! -f "$CREDS" ]]; then + (umask 027 && printf 'telegraf:\n postgres_creds: {}\n' > "$CREDS") || return 1 + chown socore:socore "$CREDS" 2>/dev/null || true + chmod 640 "$CREDS" || return 1 + fi +} + +OP=$1 +MID=$2 +[[ -z "$OP" || -z "$MID" ]] && usage + +case "$OP" in + add) + SAFE=$(echo "$MID" | tr '.-' '__' | tr '[:upper:]' '[:lower:]') + seed_creds_file || exit 1 + if so-yaml.py get -r "$CREDS" "telegraf.postgres_creds.${MID}.user" >/dev/null 2>&1; then + exit 0 + fi + PASS=$(tr -dc 'A-Za-z0-9~!@#^&*()_=+[]|;:,.<>?-' < /dev/urandom | head -c 72) + so-yaml.py replace "$CREDS" "telegraf.postgres_creds.${MID}.user" "so_telegraf_${SAFE}" >/dev/null + so-yaml.py replace "$CREDS" "telegraf.postgres_creds.${MID}.pass" "$PASS" >/dev/null + ;; + remove) + [[ -f "$CREDS" ]] || exit 0 + so-yaml.py remove "$CREDS" "telegraf.postgres_creds.${MID}" >/dev/null 2>&1 || true + ;; + *) + usage + ;; +esac diff --git a/salt/manager/tools/sbin/so-yaml.py b/salt/manager/tools/sbin/so-yaml.py index 79dcfcac0..d0d5209f9 100755 --- a/salt/manager/tools/sbin/so-yaml.py +++ b/salt/manager/tools/sbin/so-yaml.py @@ -39,9 +39,16 @@ def showUsage(args): def loadYaml(filename): - file = open(filename, "r") - content = file.read() - return yaml.safe_load(content) + try: + with open(filename, "r") as file: + content = file.read() + return yaml.safe_load(content) + except FileNotFoundError: + print(f"File not found: {filename}", file=sys.stderr) + sys.exit(1) + except Exception as e: + print(f"Error reading file {filename}: {e}", file=sys.stderr) + sys.exit(1) def writeYaml(filename, content): @@ -285,7 +292,8 @@ def add(args): def removeKey(content, key): pieces = key.split(".", 1) if len(pieces) > 1: - removeKey(content[pieces[0]], pieces[1]) + if pieces[0] in content: + removeKey(content[pieces[0]], pieces[1]) else: content.pop(key, None) diff --git a/salt/manager/tools/sbin/so-yaml_test.py b/salt/manager/tools/sbin/so-yaml_test.py index 2da8a0be9..56581f7e3 100644 --- a/salt/manager/tools/sbin/so-yaml_test.py +++ b/salt/manager/tools/sbin/so-yaml_test.py @@ -973,3 +973,21 @@ class TestReplaceListObject(unittest.TestCase): expected = "key1:\n- id: '1'\n status: updated\n- id: '2'\n status: inactive\n" self.assertEqual(actual, expected) + + +class TestLoadYaml(unittest.TestCase): + + def test_load_yaml_missing_file(self): + with patch('sys.exit', new=MagicMock()) as sysmock: + with patch('sys.stderr', new=StringIO()) as mock_stderr: + soyaml.loadYaml("/tmp/so-yaml_test-does-not-exist.yaml") + sysmock.assert_called_with(1) + self.assertIn("File not found:", mock_stderr.getvalue()) + + def test_load_yaml_read_error(self): + with patch('sys.exit', new=MagicMock()) as sysmock: + with patch('sys.stderr', new=StringIO()) as mock_stderr: + with patch('builtins.open', side_effect=PermissionError("denied")): + soyaml.loadYaml("/tmp/so-yaml_test-unreadable.yaml") + sysmock.assert_called_with(1) + self.assertIn("Error reading file", mock_stderr.getvalue()) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index d25153863..3bec13716 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -24,6 +24,14 @@ BACKUPTOPFILE=/opt/so/saltstack/default/salt/top.sls.backup SALTUPGRADED=false SALT_CLOUD_INSTALLED=false SALT_CLOUD_CONFIGURED=false +# Check if salt-cloud is installed +if rpm -q salt-cloud &>/dev/null; then + SALT_CLOUD_INSTALLED=true +fi +# Check if salt-cloud is configured +if [[ -f /etc/salt/cloud.profiles.d/socloud.conf ]]; then + SALT_CLOUD_CONFIGURED=true +fi # used to display messages to the user at the end of soup declare -a FINAL_MESSAGE_QUEUE=() @@ -363,6 +371,7 @@ preupgrade_changes() { echo "Checking to see if changes are needed." [[ "$INSTALLEDVERSION" =~ ^2\.4\.21[0-9]+$ ]] && up_to_3.0.0 + [[ "$INSTALLEDVERSION" == "3.0.0" ]] && up_to_3.1.0 true } @@ -371,6 +380,7 @@ postupgrade_changes() { echo "Running post upgrade processes." [[ "$POSTVERSION" =~ ^2\.4\.21[0-9]+$ ]] && post_to_3.0.0 + [[ "$POSTVERSION" == "3.0.0" ]] && post_to_3.1.0 true } @@ -445,7 +455,6 @@ migrate_pcap_to_suricata() { } up_to_3.0.0() { - determine_elastic_agent_upgrade migrate_pcap_to_suricata INSTALLEDVERSION=3.0.0 @@ -469,6 +478,249 @@ post_to_3.0.0() { ### 3.0.0 End ### +### 3.1.0 Scripts ### + +elasticsearch_backup_index_templates() { + echo "Backing up current elasticsearch index templates in /opt/so/conf/elasticsearch/templates/index/ to /nsm/backup/3.0.0_elasticsearch_index_templates.tar.gz" + tar -czf /nsm/backup/3.0.0_elasticsearch_index_templates.tar.gz -C /opt/so/conf/elasticsearch/templates/index/ . +} + +elasticfleet_set_agent_logging_level_warn() { + . /usr/sbin/so-elastic-fleet-common + + local current_agent_policies + if ! current_agent_policies=$(fleet_api "agent_policies?perPage=1000"); then + echo "Warning: unable to retrieve Fleet agent policies" + return 0 + fi + + # Only updating policies that are within Security Onion defaults and do not already have any user configured advanced_settings. + local policies_to_update + policies_to_update=$(jq -c ' + .items[] + | select(has("advanced_settings") | not) + | select( + .id == "so-grid-nodes_general" + or .id == "so-grid-nodes_heavy" + or .id == "endpoints-initial" + or (.id | startswith("FleetServer_")) + ) + ' <<< "$current_agent_policies") + + if [[ -z "$policies_to_update" ]]; then + return 0 + fi + + while IFS= read -r policy; do + [[ -z "$policy" ]] && continue + + local policy_id policy_name policy_namespace + policy_id=$(jq -r '.id' <<< "$policy") + policy_name=$(jq -r '.name' <<< "$policy") + policy_namespace=$(jq -r '.namespace' <<< "$policy") + + local update_logging + update_logging=$(jq -n \ + --arg name "$policy_name" \ + --arg namespace "$policy_namespace" \ + '{name: $name, namespace: $namespace, advanced_settings: {agent_logging_level: "warning"}}' + ) + + echo "Setting elastic agent_logging_level to warning on policy '$policy_name' ($policy_id)." + if ! fleet_api "agent_policies/$policy_id" -XPUT -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$update_logging" >/dev/null; then + echo " warning: failed to update agent policy '$policy_name' ($policy_id)" >&2 + fi + done <<< "$policies_to_update" +} + +check_transform_health_and_reauthorize() { + . /usr/sbin/so-elastic-fleet-common + + echo "Checking integration transform jobs for unhealthy / unauthorized status..." + + local transforms_doc stats_doc installed_doc + if ! transforms_doc=$(so-elasticsearch-query "_transform/_all?size=1000" --fail --retry 3 --retry-delay 5 2>/dev/null); then + echo "Unable to query for transform jobs, skipping reauthorization." + return 0 + fi + if ! stats_doc=$(so-elasticsearch-query "_transform/_all/_stats?size=1000" --fail --retry 3 --retry-delay 5 2>/dev/null); then + echo "Unable to query for transform job stats, skipping reauthorization." + return 0 + fi + if ! installed_doc=$(fleet_api "epm/packages/installed?perPage=500"); then + echo "Unable to list installed Fleet packages, skipping reauthorization." + return 0 + fi + + # Get all transforms that meet the following + # - unhealthy (any non-green health status) + # - metadata has run_as_kibana_system: false (this fix is specific to transforms started prior to Kibana 9.3.3) + # - are not orphaned (integration is not somehow missing/corrupt/uninstalled) + local tmp_transforms tmp_stats tmp_installed + tmp_transforms=$(mktemp) + tmp_stats=$(mktemp) + tmp_installed=$(mktemp) + + echo "$transforms_doc" > "$tmp_transforms" + echo "$stats_doc" > "$tmp_stats" + echo "$installed_doc" > "$tmp_installed" + + local unhealthy_transforms + unhealthy_transforms=$(jq -c -n \ + --slurpfile t "$tmp_transforms" \ + --slurpfile s "$tmp_stats" \ + --slurpfile i "$tmp_installed" ' + ($i[0].items | map({key: .name, value: .version}) | from_entries) as $pkg_ver + | ($s[0].transforms | map({key: .id, value: .health.status}) | from_entries) as $health + | [ $t[0].transforms[] + | select(._meta.run_as_kibana_system == false) + | select(($health[.id] // "unknown") != "green") + | {id, pkg: ._meta.package.name, ver: ($pkg_ver[._meta.package.name])} + ] + | if length == 0 then empty else . end + | (map(select(.ver == null)) | map({orphan: .id})[]), + (map(select(.ver != null)) + | group_by(.pkg) + | map({pkg: .[0].pkg, ver: .[0].ver, transformIds: map(.id)})[]) + ') + + if [[ -z "$unhealthy_transforms" ]]; then + return 0 + fi + + local unhealthy_count + unhealthy_count=$(jq -s '[.[].transformIds? // empty | .[]] | length' <<< "$unhealthy_transforms") + echo "Found $unhealthy_count transform(s) needing reauthorization." + + local total_failures=0 + while IFS= read -r transform; do + [[ -z "$transform" ]] && continue + if jq -e 'has("orphan")' <<< "$transform" >/dev/null 2>&1; then + echo "Skipping transform not owned by any installed Fleet package: $(jq -r '.orphan' <<< "$transform")" + continue + fi + + local pkg ver body resp + pkg=$(jq -r '.pkg' <<< "$transform") + ver=$(jq -r '.ver' <<< "$transform") + body=$(jq -c '{transforms: (.transformIds | map({transformId: .}))}' <<< "$transform") + + echo "Reauthorizing transform(s) for ${pkg}-${ver}..." + resp=$(fleet_api "epm/packages/${pkg}/${ver}/transforms/authorize" \ + -XPOST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' \ + -d "$body") || { echo "Could not reauthorize transform(s) for ${pkg}-${ver}"; continue; } + + (( total_failures += $(jq 'map(select(.success != true)) | length' <<< "$resp" 2>/dev/null) )) + done <<< "$unhealthy_transforms" + + rm -f "$tmp_transforms" "$tmp_stats" "$tmp_installed" + + if [[ "$total_failures" -gt 0 ]]; then + echo "Some transform(s) failed to reauthorize." + fi +} + +ensure_postgres_local_pillar() { + # Postgres was added as a service after 3.0.0, so the new pillar/top.sls + # references postgres.soc_postgres / postgres.adv_postgres unconditionally. + # Managers upgrading from 3.0.0 have no /opt/so/saltstack/local/pillar/postgres/ + # (make_some_dirs only runs at install time), so the stubs must be created + # here before salt-master restarts against the new top.sls. + echo "Ensuring postgres local pillar stubs exist." + local dir=/opt/so/saltstack/local/pillar/postgres + mkdir -p "$dir" + [[ -f "$dir/soc_postgres.sls" ]] || touch "$dir/soc_postgres.sls" + [[ -f "$dir/adv_postgres.sls" ]] || touch "$dir/adv_postgres.sls" + chown -R socore:socore "$dir" +} + +ensure_postgres_secret() { + # On a fresh install, generate_passwords + secrets_pillar seed + # secrets:postgres_pass in /opt/so/saltstack/local/pillar/secrets.sls. That + # code path is skipped on upgrade (secrets.sls already exists from 3.0.0 + # with import_pass/influx_pass but no postgres_pass), so the postgres + # container's POSTGRES_PASSWORD_FILE and SOC's PG_ADMIN_PASS would be empty + # after highstate. Generate one now if missing. + local secrets_file=/opt/so/saltstack/local/pillar/secrets.sls + if [[ ! -f "$secrets_file" ]]; then + echo "WARNING: $secrets_file missing; skipping postgres_pass backfill." + return 0 + fi + if so-yaml.py get -r "$secrets_file" secrets.postgres_pass >/dev/null 2>&1; then + echo "secrets.postgres_pass already set; leaving as-is." + return 0 + fi + echo "Seeding secrets.postgres_pass in $secrets_file." + so-yaml.py add "$secrets_file" secrets.postgres_pass "$(get_random_value)" + chown socore:socore "$secrets_file" +} + +rename_strelka_scan_lnk() { + echo "Renaming strelka pillar ScanLNK to ScanLnk." + local STRELKA_FILE=/opt/so/saltstack/local/pillar/strelka/soc_strelka.sls + local MINIONDIR=/opt/so/saltstack/local/pillar/minions + local OLD_KEY=strelka.backend.config.backend.scanners.ScanLNK + local NEW_KEY=strelka.backend.config.backend.scanners.ScanLnk + local TMP_VALUE_FILE + TMP_VALUE_FILE=$(mktemp) + + for pillar_file in "$STRELKA_FILE" "$MINIONDIR"/*.sls; do + [[ -f "$pillar_file" ]] || continue + # Skip if ScanLNK doesn't exist + so-yaml.py get "$pillar_file" "$OLD_KEY" > "$TMP_VALUE_FILE" 2>/dev/null || continue + echo "Found 'ScanLNK' key in $pillar_file. Renaming to 'ScanLnk'." + so-yaml.py add "$pillar_file" "$NEW_KEY" "file:$TMP_VALUE_FILE" + so-yaml.py remove "$pillar_file" "$OLD_KEY" + done + + rm -f "$TMP_VALUE_FILE" +} + +up_to_3.1.0() { + ensure_postgres_local_pillar + ensure_postgres_secret + determine_elastic_agent_upgrade + elasticsearch_backup_index_templates + # Clear existing component template state file. + rm -f /opt/so/state/esfleet_component_templates.json + rename_strelka_scan_lnk + + INSTALLEDVERSION=3.1.0 +} + +post_to_3.1.0() { + /usr/sbin/so-kibana-space-defaults + # ensure manager has new version of socloud.conf + if [[ $SALT_CLOUD_CONFIGURED == true ]]; then + salt-call state.apply salt.cloud.config concurrent=True + fi + + # Backfill the Telegraf creds pillar for every accepted minion. so-telegraf-cred + # add is idempotent — it no-ops when an entry already exists — so this is safe + # to run on every soup. The subsequent state.apply creates/updates the matching + # Postgres roles from the reconciled pillar. + echo "Reconciling Telegraf Postgres creds for accepted minions." + for mid in $(salt-key --out=json --list=accepted 2>/dev/null | jq -r '.minions[]?' 2>/dev/null); do + [[ -n "$mid" ]] || continue + /usr/sbin/so-telegraf-cred add "$mid" || echo " warning: so-telegraf-cred add $mid failed" >&2 + done + # Run through the master (not --local) so state compilation uses the + # master's configured file_roots; the manager's /etc/salt/minion has no + # file_roots of its own and --local would fail with "No matching sls found". + salt-call state.apply postgres.telegraf_users queue=True || true + + # Update default agent policies to use logging level warn. + elasticfleet_set_agent_logging_level_warn || true + + # Check for unhealthy / unauthorized integration transform jobs and attempt reauthorizations + check_transform_health_and_reauthorize || true + + POSTVERSION=3.1.0 +} + +### 3.1.0 End ### + + repo_sync() { echo "Sync the local repo." su socore -c '/usr/sbin/so-repo-sync' || fail "Unable to complete so-repo-sync." @@ -636,15 +888,6 @@ upgrade_check_salt() { upgrade_salt() { echo "Performing upgrade of Salt from $INSTALLEDSALTVERSION to $NEWSALTVERSION." echo "" - # Check if salt-cloud is installed - if rpm -q salt-cloud &>/dev/null; then - SALT_CLOUD_INSTALLED=true - fi - # Check if salt-cloud is configured - if [[ -f /etc/salt/cloud.profiles.d/socloud.conf ]]; then - SALT_CLOUD_CONFIGURED=true - fi - echo "Removing yum versionlock for Salt." echo "" yum versionlock delete "salt" @@ -728,12 +971,15 @@ verify_es_version_compatibility() { local is_active_intermediate_upgrade=1 # supported upgrade paths for SO-ES versions declare -A es_upgrade_map=( + ["8.18.4"]="8.18.6 8.18.8 9.0.8" + ["8.18.6"]="8.18.8 9.0.8" ["8.18.8"]="9.0.8" + ["9.0.8"]="9.3.3" ) # Elasticsearch MUST upgrade through these versions declare -A es_to_so_version=( - ["8.18.8"]="2.4.190-20251024" + ["9.0.8"]="3.0.0-20260331" ) # Get current Elasticsearch version @@ -745,26 +991,182 @@ verify_es_version_compatibility() { exit 160 fi - if ! target_es_version_raw=$(so-yaml.py get $UPDATE_DIR/salt/elasticsearch/defaults.yaml elasticsearch.version); then - # so-yaml.py failed to get the ES version from upgrade versions elasticsearch/defaults.yaml file. Likely they are upgrading to an SO version older than 2.4.110 prior to the ES version pinning and should be OKAY to continue with the upgrade. + if ! target_es_version=$(so-yaml.py get -r $UPDATE_DIR/salt/elasticsearch/defaults.yaml elasticsearch.version); then + echo "Couldn't determine the target Elasticsearch version (post soup version) to ensure compatibility with current Elasticsearch version. Exiting" - # if so-yaml.py failed to get the ES version AND the version we are upgrading to is newer than 2.4.110 then we should bail - if [[ $(cat $UPDATE_DIR/VERSION | cut -d'.' -f3) > 110 ]]; then - echo "Couldn't determine the target Elasticsearch version (post soup version) to ensure compatibility with current Elasticsearch version. Exiting" + exit 160 + fi - exit 160 + compatible_es_versions="$target_es_version" + for current_version in "${!es_upgrade_map[@]}"; do + # shellcheck disable=SC2076 + if [[ " ${es_upgrade_map[$current_version]} " =~ " $target_es_version " ]]; then + compatible_es_versions+=" $current_version" + fi + done + + # Check if the given ES version can directly upgrade to the target ES version. Used to assist with catching lagging nodes during the upgrade process + es_version_can_upgrade_to_target() { + local current_version="$1" + # shellcheck disable=SC2076 + if [[ -n "$current_version" && " $compatible_es_versions " =~ " $current_version " ]]; then + return 0 fi - # allow upgrade to version < 2.4.110 without checking ES version compatibility - return 0 - else - target_es_version=$(sed -n '1p' <<< "$target_es_version_raw") + return 1 + } + + # Gather Elasticsearch cluster version info and verify that each node in the cluster is running a version compatible with the target ES version. + verify_searchnodes_es_target_compatibility() { + local retries=20 + local retry_count=0 + local delay=180 + local expected_es_nodes searchnode_minions attempt + local searchnode_discovery_success=false + SEARCHNODE_ES_VERSIONS="" + + for attempt in {1..3}; do + if searchnode_minions=$(set -o pipefail; salt-key --out=json --list=accepted 2> /dev/null | jq -r '.minions[]? | select(endswith("searchnode"))'); then + searchnode_discovery_success=true + break + fi + + echo "Failed to retrieve grid searchnodes via salt-key... Retrying in 30 seconds. Attempt $attempt of 3." + sleep 30 + done + + if [[ "$searchnode_discovery_success" != "true" ]]; then + echo "Failed to retrieve grid searchnodes via salt-key." + return 1 + fi + + # Always add node running soup to expected es nodes + expected_es_nodes="${MINIONID%_*}" + while IFS= read -r searchnode_minion; do + [[ -z "$searchnode_minion" ]] && continue + expected_es_nodes+=$'\n'"${searchnode_minion%_searchnode}" + done <<< "$searchnode_minions" + + while [[ $retry_count -lt $retries ]]; do + SEARCHNODE_ES_VERSIONS=$(so-elasticsearch-query _nodes/_all/version --retry 5 --retry-delay 10 --fail 2>&1) + local exit_status=$? + + if [[ $exit_status -ne 0 ]]; then + echo "Failed to retrieve Elasticsearch versions from searchnodes... Retrying in $delay seconds. Attempt $((retry_count + 1)) of $retries." + ((retry_count++)) + sleep $delay + continue + fi + + local all_searchnodes_compatible=true + while IFS=$'\t' read -r node current_version; do + [[ -z "$node" ]] && continue + if ! es_version_can_upgrade_to_target "$current_version"; then + echo "Searchnode $node is running Elasticsearch $current_version, which is not directly upgradable to Elasticsearch $target_es_version." + all_searchnodes_compatible=false + fi + done < <(echo "$SEARCHNODE_ES_VERSIONS" | jq -r '.nodes | to_entries[] | [.value.name, .value.version] | @tsv') + + while IFS= read -r expected_es_node; do + [[ -z "$expected_es_node" ]] && continue + if ! echo "$SEARCHNODE_ES_VERSIONS" | jq -e --arg node "$expected_es_node" '.nodes | to_entries | any(.value.name == $node)' > /dev/null; then + echo "Searchnode $expected_es_node did not report an Elasticsearch version. It may be offline or still upgrading." + all_searchnodes_compatible=false + fi + done <<< "$expected_es_nodes" + + if [[ "$all_searchnodes_compatible" == true ]]; then + echo "All Searchnodes are upgradable to Elasticsearch $target_es_version." + return 0 + fi + + echo "One or more Searchnodes cannot upgrade directly to Elasticsearch $target_es_version. Rechecking in $delay seconds. Attempt $((retry_count + 1)) of $retries." + ((retry_count++)) + sleep $delay + done + + return 1 + } + + # Gather heavynode version info and verify that each node is running a version compatible with the target ES version. + verify_heavynodes_es_target_compatibility() { + local heavynode_minions attempt + local retries=20 + local retry_count=0 + local delay=180 + local heavynode_discovery_success=false + HEAVYNODE_ES_VERSIONS="" + + for attempt in {1..3}; do + if heavynode_minions=$(set -o pipefail; salt-key --out=json --list=accepted 2> /dev/null | jq -r '.minions[]? | select(endswith("heavynode"))'); then + heavynode_discovery_success=true + break + fi + + echo "Failed to retrieve grid heavynodes via salt-key... Retrying in 30 seconds. Attempt $attempt of 3." + sleep 30 + done + + if [[ "$heavynode_discovery_success" != "true" ]]; then + echo "Failed to retrieve grid heavynodes via salt-key." + return 1 + fi + + if [[ -z "$heavynode_minions" ]]; then + echo "No heavynodes detected. Skipping heavynode Elasticsearch version compatibility check." + return 0 + fi + + while [[ $retry_count -lt $retries ]]; do + HEAVYNODE_ES_VERSIONS=$(salt -C 'G@role:so-heavynode' cmd.run 'set -o pipefail; so-elasticsearch-query / --retry 5 --retry-delay 10 | jq -er ".version.number"' shell=/bin/bash --out=json 2> /dev/null) + local exit_status=$? + + if [[ $exit_status -ne 0 ]]; then + echo "Failed to retrieve Elasticsearch version from one or more heavynodes... Retrying in $delay seconds. Attempt $((retry_count + 1)) of $retries." + ((retry_count++)) + sleep $delay + continue + fi + + local all_heavynodes_compatible=true + while IFS=$'\t' read -r node current_version; do + [[ -z "$node" ]] && continue + if ! es_version_can_upgrade_to_target "$current_version"; then + echo "Heavynode $node is running Elasticsearch $current_version, which is not directly upgradable to Elasticsearch $target_es_version." + all_heavynodes_compatible=false + fi + done < <(echo "$HEAVYNODE_ES_VERSIONS" | jq -r 'to_entries[] | [.key, .value] | @tsv') + + while IFS= read -r heavynode_minion; do + [[ -z "$heavynode_minion" ]] && continue + if ! echo "$HEAVYNODE_ES_VERSIONS" | jq -e --arg minion "$heavynode_minion" 'has($minion)' > /dev/null; then + echo "Heavynode $heavynode_minion did not report an Elasticsearch version. It may be offline or still upgrading." + all_heavynodes_compatible=false + fi + done <<< "$heavynode_minions" + + if [[ "$all_heavynodes_compatible" == true ]]; then + echo -e "\nAll heavynodes can upgrade to Elasticsearch $target_es_version." + return 0 + fi + + echo "One or more heavynodes cannot upgrade directly to Elasticsearch $target_es_version. Rechecking in $delay seconds. Attempt $((retry_count + 1)) of $retries." + ((retry_count++)) + sleep $delay + done + + return 1 + } + + if [[ ! -f "$es_verification_script" ]]; then + create_intermediate_upgrade_verification_script "$es_verification_script" fi for statefile in "${es_required_version_statefile_base}"-*; do [[ -f $statefile ]] || continue - local es_required_version_statefile_value=$(cat "$statefile") + local es_required_version_statefile_value + es_required_version_statefile_value=$(cat "$statefile") if [[ "$es_required_version_statefile_value" == "$target_es_version" ]]; then echo "Intermediate upgrade to ES $target_es_version is in progress. Skipping Elasticsearch version compatibility check." @@ -773,19 +1175,14 @@ verify_es_version_compatibility() { fi # use sort to check if es_required_statefile_value is < the current es_version. - if [[ "$(printf '%s\n' $es_required_version_statefile_value $es_version | sort -V | head -n1)" == "$es_required_version_statefile_value" ]]; then + if [[ "$(printf '%s\n' "$es_required_version_statefile_value" "$es_version" | sort -V | head -n1)" == "$es_required_version_statefile_value" ]]; then rm -f "$statefile" continue fi - if [[ ! -f "$es_verification_script" ]]; then - create_intermediate_upgrade_verification_script "$es_verification_script" - fi - echo -e "\n##############################################################################################################################\n" echo "A previously required intermediate Elasticsearch upgrade was detected. Verifying that all Searchnodes/Heavynodes have successfully upgraded Elasticsearch to $es_required_version_statefile_value before proceeding with soup to avoid potential data loss! This command can take up to an hour to complete." - timeout --foreground 4000 bash "$es_verification_script" "$es_required_version_statefile_value" "$statefile" - if [[ $? -ne 0 ]]; then + if ! timeout --foreground 4000 bash "$es_verification_script" "$es_required_version_statefile_value" "$statefile"; then echo -e "\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n" echo "A previous required intermediate Elasticsearch upgrade to $es_required_version_statefile_value has yet to successfully complete across the grid. Please allow time for all Searchnodes/Heavynodes to have upgraded Elasticsearch to $es_required_version_statefile_value before running soup again to avoid potential data loss!" @@ -802,7 +1199,28 @@ verify_es_version_compatibility() { return 0 fi + # shellcheck disable=SC2076 # Do not want a regex here eg usage " 8.18.8 9.0.8 " =~ " 9.0.8 " if [[ " ${es_upgrade_map[$es_version]} " =~ " $target_es_version " || "$es_version" == "$target_es_version" ]]; then + if ! verify_searchnodes_es_target_compatibility || ! verify_heavynodes_es_target_compatibility; then + echo -e "\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n" + + echo "One or more Searchnode(s)/Heavynode(s) cannot upgrade directly to Elasticsearch $target_es_version. This can happen with soups that include Elasticsearch upgrades being run in quick succession. Typically, this will resolve itself as the grid synchronizes. Please allow time for all Searchnodes/Heavynodes to have upgraded Elasticsearch to a compatible version with $target_es_version before running soup again to avoid potential data loss!" + + if [[ -n "$HEAVYNODE_ES_VERSIONS" ]]; then + echo "Current heavynode Elasticsearch versions:" + echo "$HEAVYNODE_ES_VERSIONS" | jq '.' + fi + + if [[ -n "$SEARCHNODE_ES_VERSIONS" ]]; then + echo "Current searchnode Elasticsearch versions:" + echo "$SEARCHNODE_ES_VERSIONS" | jq '.nodes | to_entries | map({(.value.name): .value.version}) | sort | add' + fi + + echo -e "\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n" + + exit 161 + fi + # supported upgrade return 0 else @@ -810,7 +1228,7 @@ verify_es_version_compatibility() { if [[ -z "$compatible_versions" ]]; then # If current ES version is not explicitly defined in the upgrade map, we know they have an intermediate upgrade to do. # We default to the lowest ES version defined in es_to_so_version as $first_es_required_version - local first_es_required_version=$(printf '%s\n' "${!es_to_so_version[@]}" | sort -V | head -n1) + first_es_required_version=$(printf '%s\n' "${!es_to_so_version[@]}" | sort -V | head -n1) next_step_so_version=${es_to_so_version[$first_es_required_version]} required_es_upgrade_version="$first_es_required_version" else @@ -829,7 +1247,7 @@ verify_es_version_compatibility() { if [[ $is_airgap -eq 0 ]]; then run_airgap_intermediate_upgrade else - if [[ ! -z $ISOLOC ]]; then + if [[ -n $ISOLOC ]]; then originally_requested_iso_location="$ISOLOC" fi # Make sure ISOLOC is not set. Network installs that used soup -f would have ISOLOC set. @@ -861,7 +1279,8 @@ wait_for_salt_minion_with_restart() { } run_airgap_intermediate_upgrade() { - local originally_requested_so_version=$(cat $UPDATE_DIR/VERSION) + local originally_requested_so_version + originally_requested_so_version=$(cat "$UPDATE_DIR/VERSION") # preserve ISOLOC value, so we can try to use it post intermediate upgrade local originally_requested_iso_location="$ISOLOC" @@ -873,7 +1292,8 @@ run_airgap_intermediate_upgrade() { while [[ -z "$next_iso_location" ]] || [[ ! -f "$next_iso_location" && ! -b "$next_iso_location" ]]; do # List removable devices if any are present - local removable_devices=$(lsblk -no PATH,SIZE,TYPE,MOUNTPOINTS,RM | awk '$NF==1') + local removable_devices + removable_devices=$(lsblk -no PATH,SIZE,TYPE,MOUNTPOINTS,RM | awk '$NF==1') if [[ -n "$removable_devices" ]]; then echo "PATH SIZE TYPE MOUNTPOINTS RM" echo "$removable_devices" @@ -894,21 +1314,21 @@ run_airgap_intermediate_upgrade() { echo "Using $next_iso_location for required intermediary upgrade." exec bash < /dev/null 2>&1 + - identifier: so_postgres_backup + - user: root + +{% else %} + +{{sls}}_state_not_allowed: + test.fail_without_changes: + - name: {{sls}}_state_not_allowed + +{% endif %} diff --git a/salt/postgres/enabled.sls b/salt/postgres/enabled.sls new file mode 100644 index 000000000..20d256ae8 --- /dev/null +++ b/salt/postgres/enabled.sls @@ -0,0 +1,109 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{% from 'allowed_states.map.jinja' import allowed_states %} +{% if sls.split('.')[0] in allowed_states %} +{% from 'vars/globals.map.jinja' import GLOBALS %} +{% from 'docker/docker.map.jinja' import DOCKERMERGED %} +{% set SO_POSTGRES_USER = salt['pillar.get']('postgres:auth:users:so_postgres_user:user', 'so_postgres') %} + +include: + - postgres.auth + - postgres.ssl + - postgres.config + - postgres.sostatus + - postgres.telegraf_users + +so-postgres: + docker_container.running: + - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-postgres:{{ GLOBALS.so_version }} + - hostname: so-postgres + - networks: + - sobridge: + - ipv4_address: {{ DOCKERMERGED.containers['so-postgres'].ip }} + - port_bindings: + {% for BINDING in DOCKERMERGED.containers['so-postgres'].port_bindings %} + - {{ BINDING }} + {% endfor %} + - environment: + - POSTGRES_DB=securityonion + # Passwords are delivered via mounted 0600 secret files, not plaintext env vars. + # The upstream postgres image resolves POSTGRES_PASSWORD_FILE; entrypoint.sh and + # init-db.sh resolve SO_POSTGRES_PASS_FILE the same way. + - POSTGRES_PASSWORD_FILE=/run/secrets/postgres_password + - SO_POSTGRES_USER={{ SO_POSTGRES_USER }} + - SO_POSTGRES_PASS_FILE=/run/secrets/so_postgres_pass + {% if DOCKERMERGED.containers['so-postgres'].extra_env %} + {% for XTRAENV in DOCKERMERGED.containers['so-postgres'].extra_env %} + - {{ XTRAENV }} + {% endfor %} + {% endif %} + - binds: + - /opt/so/log/postgres/:/log:rw + - /nsm/postgres:/var/lib/postgresql/data:rw + - /opt/so/conf/postgres/postgresql.conf:/conf/postgresql.conf:ro + - /opt/so/conf/postgres/pg_hba.conf:/conf/pg_hba.conf:ro + - /opt/so/conf/postgres/secrets:/run/secrets:ro + - /opt/so/conf/postgres/init/init-db.sh:/docker-entrypoint-initdb.d/init-db.sh:ro + - /etc/pki/postgres.crt:/conf/postgres.crt:ro + - /etc/pki/postgres.key:/conf/postgres.key:ro + - /etc/pki/tls/certs/intca.crt:/conf/ca.crt:ro + {% if DOCKERMERGED.containers['so-postgres'].custom_bind_mounts %} + {% for BIND in DOCKERMERGED.containers['so-postgres'].custom_bind_mounts %} + - {{ BIND }} + {% endfor %} + {% endif %} + {% if DOCKERMERGED.containers['so-postgres'].extra_hosts %} + - extra_hosts: + {% for XTRAHOST in DOCKERMERGED.containers['so-postgres'].extra_hosts %} + - {{ XTRAHOST }} + {% endfor %} + {% endif %} + {% if DOCKERMERGED.containers['so-postgres'].ulimits %} + - ulimits: + {% for ULIMIT in DOCKERMERGED.containers['so-postgres'].ulimits %} + - {{ ULIMIT.name }}={{ ULIMIT.soft }}:{{ ULIMIT.hard }} + {% endfor %} + {% endif %} + - watch: + - file: postgresconf + - file: postgreshba + - file: postgresinitdb + - file: postgres_super_secret + - file: postgres_app_secret + - x509: postgres_crt + - x509: postgres_key + - require: + - file: postgresconf + - file: postgreshba + - file: postgresinitdb + - file: postgres_super_secret + - file: postgres_app_secret + - x509: postgres_crt + - x509: postgres_key + +delete_so-postgres_so-status.disabled: + file.uncomment: + - name: /opt/so/conf/so-status/so-status.conf + - regex: ^so-postgres$ + +so_postgres_backup: + cron.present: + - name: /usr/sbin/so-postgres-backup > /dev/null 2>&1 + - identifier: so_postgres_backup + - user: root + - minute: '5' + - hour: '0' + - daymonth: '*' + - month: '*' + - dayweek: '*' + +{% else %} + +{{sls}}_state_not_allowed: + test.fail_without_changes: + - name: {{sls}}_state_not_allowed + +{% endif %} diff --git a/salt/postgres/files/init-db.sh b/salt/postgres/files/init-db.sh new file mode 100644 index 000000000..e28b11f0f --- /dev/null +++ b/salt/postgres/files/init-db.sh @@ -0,0 +1,34 @@ +#!/bin/bash +set -e + +# Create or update application user for SOC platform access +# This script runs on first database initialization via docker-entrypoint-initdb.d +# The password is properly escaped to handle special characters +if [ -z "${SO_POSTGRES_PASS:-}" ] && [ -n "${SO_POSTGRES_PASS_FILE:-}" ] && [ -r "$SO_POSTGRES_PASS_FILE" ]; then + SO_POSTGRES_PASS="$(< "$SO_POSTGRES_PASS_FILE")" +fi +psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL + DO \$\$ + BEGIN + IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = '${SO_POSTGRES_USER}') THEN + EXECUTE format('CREATE ROLE %I WITH LOGIN PASSWORD %L', '${SO_POSTGRES_USER}', '${SO_POSTGRES_PASS}'); + ELSE + EXECUTE format('ALTER ROLE %I WITH PASSWORD %L', '${SO_POSTGRES_USER}', '${SO_POSTGRES_PASS}'); + END IF; + END + \$\$; + GRANT ALL PRIVILEGES ON DATABASE "$POSTGRES_DB" TO "$SO_POSTGRES_USER"; + -- Lock the SOC database down at the connect layer; PUBLIC gets CONNECT + -- by default, which would let per-minion telegraf roles open sessions + -- here. They have no schema/table grants inside so reads fail, but + -- revoking CONNECT closes the soft edge entirely. + REVOKE CONNECT ON DATABASE "$POSTGRES_DB" FROM PUBLIC; + GRANT CONNECT ON DATABASE "$POSTGRES_DB" TO "$SO_POSTGRES_USER"; +EOSQL + +# Bootstrap the Telegraf metrics database. Per-minion roles + schemas are +# reconciled on every state.apply by postgres/telegraf_users.sls; this block +# only ensures the shared database exists on first initialization. +if ! psql -U "$POSTGRES_USER" -tAc "SELECT 1 FROM pg_database WHERE datname='so_telegraf'" | grep -q 1; then + psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -c "CREATE DATABASE so_telegraf" +fi diff --git a/salt/postgres/files/pg_hba.conf b/salt/postgres/files/pg_hba.conf new file mode 100644 index 000000000..e7d31c05f --- /dev/null +++ b/salt/postgres/files/pg_hba.conf @@ -0,0 +1,16 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. +# +# Managed by Salt — do not edit by hand. +# Client authentication config: only local (Unix socket) connections and TLS-wrapped TCP +# connections are accepted. Plain-text `host ...` lines are intentionally omitted so a +# misconfigured client with sslmode=disable cannot negotiate a cleartext session. + +# Local connections (Unix socket, container-internal) use peer/trust. +local all all trust + +# TCP connections MUST use TLS (hostssl) and authenticate with SCRAM. +hostssl all all 0.0.0.0/0 scram-sha-256 +hostssl all all ::/0 scram-sha-256 diff --git a/salt/postgres/files/postgresql.conf.jinja b/salt/postgres/files/postgresql.conf.jinja new file mode 100644 index 000000000..2ddc52a51 --- /dev/null +++ b/salt/postgres/files/postgresql.conf.jinja @@ -0,0 +1,8 @@ +{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one + or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at + https://securityonion.net/license; you may not use this file except in compliance with the + Elastic License 2.0. #} + +{% for key, value in PGMERGED.config.items() %} +{{ key }} = '{{ value | string | replace("'", "''") }}' +{% endfor %} diff --git a/salt/postgres/init.sls b/salt/postgres/init.sls new file mode 100644 index 000000000..2e3c9ffb7 --- /dev/null +++ b/salt/postgres/init.sls @@ -0,0 +1,13 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{% from 'postgres/map.jinja' import PGMERGED %} + +include: +{% if PGMERGED.enabled %} + - postgres.enabled +{% else %} + - postgres.disabled +{% endif %} diff --git a/salt/postgres/map.jinja b/salt/postgres/map.jinja new file mode 100644 index 000000000..5250ca8fd --- /dev/null +++ b/salt/postgres/map.jinja @@ -0,0 +1,7 @@ +{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one + or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at + https://securityonion.net/license; you may not use this file except in compliance with the + Elastic License 2.0. #} + +{% import_yaml 'postgres/defaults.yaml' as PGDEFAULTS %} +{% set PGMERGED = salt['pillar.get']('postgres', PGDEFAULTS.postgres, merge=True) %} diff --git a/salt/postgres/soc_postgres.yaml b/salt/postgres/soc_postgres.yaml new file mode 100644 index 000000000..4b25cd4f5 --- /dev/null +++ b/salt/postgres/soc_postgres.yaml @@ -0,0 +1,89 @@ +postgres: + enabled: + description: Whether the PostgreSQL database container is enabled on this grid. Backs the assistant store and the Telegraf metrics database. + forcedType: bool + readonly: True + helpLink: influxdb + telegraf: + retention_days: + description: Number of days of Telegraf metrics to keep in the so_telegraf database. Older partitions are dropped hourly by pg_partman. + forcedType: int + helpLink: postgres + config: + max_connections: + description: Maximum number of concurrent PostgreSQL connections. + forcedType: int + global: True + helpLink: postgres + shared_buffers: + description: Amount of memory PostgreSQL uses for shared buffers (e.g. 256MB, 1GB). Raising this improves read cache hit rate at the cost of system RAM. + global: True + helpLink: postgres + log_min_messages: + description: Minimum severity of server messages written to the PostgreSQL log. + options: + - debug1 + - info + - notice + - warning + - error + - log + - fatal + global: True + helpLink: postgres + listen_addresses: + description: Interfaces PostgreSQL listens on. Must remain '*' so clients on the docker bridge network can connect. + global: True + advanced: True + helpLink: postgres + port: + description: TCP port PostgreSQL listens on inside the container. Firewall rules and container port mapping assume 5432. + forcedType: int + global: True + advanced: True + helpLink: postgres + ssl: + description: Whether PostgreSQL accepts TLS connections. Must remain 'on' — pg_hba.conf requires hostssl for TCP. + global: True + advanced: True + helpLink: postgres + ssl_cert_file: + description: Path (inside the container) to the TLS server certificate. Salt-managed. + global: True + advanced: True + helpLink: postgres + ssl_key_file: + description: Path (inside the container) to the TLS server private key. Salt-managed. + global: True + advanced: True + helpLink: postgres + ssl_ca_file: + description: Path (inside the container) to the CA bundle PostgreSQL uses to verify client certificates. Salt-managed. + global: True + advanced: True + helpLink: postgres + hba_file: + description: Path (inside the container) to the pg_hba.conf authentication file. Salt-managed — edit salt/postgres/files/pg_hba.conf. + global: True + advanced: True + helpLink: postgres + log_destination: + description: Where PostgreSQL writes its server log. 'stderr' routes to the container log stream. + global: True + advanced: True + helpLink: postgres + logging_collector: + description: Whether to run a separate logging collector process. Disabled because the docker log stream already captures stderr. + global: True + advanced: True + helpLink: postgres + shared_preload_libraries: + description: Comma-separated list of extensions loaded at server start. Required for pg_cron which drives pg_partman maintenance — do not remove. + global: True + advanced: True + helpLink: postgres + cron.database_name: + description: Database pg_cron schedules jobs in. Must be so_telegraf so partman maintenance runs in the right database context. + global: True + advanced: True + helpLink: postgres diff --git a/salt/postgres/sostatus.sls b/salt/postgres/sostatus.sls new file mode 100644 index 000000000..4a61af3d1 --- /dev/null +++ b/salt/postgres/sostatus.sls @@ -0,0 +1,21 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{% from 'allowed_states.map.jinja' import allowed_states %} +{% if sls.split('.')[0] in allowed_states %} + +append_so-postgres_so-status.conf: + file.append: + - name: /opt/so/conf/so-status/so-status.conf + - text: so-postgres + - unless: grep -q so-postgres /opt/so/conf/so-status/so-status.conf + +{% else %} + +{{sls}}_state_not_allowed: + test.fail_without_changes: + - name: {{sls}}_state_not_allowed + +{% endif %} diff --git a/salt/postgres/ssl.sls b/salt/postgres/ssl.sls new file mode 100644 index 000000000..4223ead34 --- /dev/null +++ b/salt/postgres/ssl.sls @@ -0,0 +1,55 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{% from 'allowed_states.map.jinja' import allowed_states %} +{% if sls.split('.')[0] in allowed_states %} +{% from 'vars/globals.map.jinja' import GLOBALS %} +{% from 'ca/map.jinja' import CA %} + +postgres_key: + x509.private_key_managed: + - name: /etc/pki/postgres.key + - keysize: 4096 + - backup: True + - new: True + {% if salt['file.file_exists']('/etc/pki/postgres.key') -%} + - prereq: + - x509: /etc/pki/postgres.crt + {%- endif %} + - retry: + attempts: 5 + interval: 30 + +postgres_crt: + x509.certificate_managed: + - name: /etc/pki/postgres.crt + - ca_server: {{ CA.server }} + - subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }} + - signing_policy: postgres + - private_key: /etc/pki/postgres.key + - CN: {{ GLOBALS.hostname }} + - days_remaining: 7 + - days_valid: 820 + - backup: True + - timeout: 30 + - retry: + attempts: 5 + interval: 30 + +postgresKeyperms: + file.managed: + - replace: False + - name: /etc/pki/postgres.key + - mode: 400 + - user: 939 + - group: 939 + +{% else %} + +{{sls}}_state_not_allowed: + test.fail_without_changes: + - name: {{sls}}_state_not_allowed + +{% endif %} diff --git a/salt/postgres/telegraf_users.sls b/salt/postgres/telegraf_users.sls new file mode 100644 index 000000000..28d9d6247 --- /dev/null +++ b/salt/postgres/telegraf_users.sls @@ -0,0 +1,104 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{% from 'allowed_states.map.jinja' import allowed_states %} +{% if sls.split('.')[0] in allowed_states %} +{% from 'vars/globals.map.jinja' import GLOBALS %} +{% from 'telegraf/map.jinja' import TELEGRAFMERGED %} + +{# postgres_wait_ready below requires `docker_container: so-postgres`, which is + declared in postgres.enabled. Include it here so state.apply postgres.telegraf_users + on its own (e.g. from orch.deploy_newnode) still has that ID in scope. Salt + de-duplicates the circular include. #} +include: + - postgres.enabled + +{% set TG_OUT = TELEGRAFMERGED.output | upper %} +{% if TG_OUT in ['POSTGRES', 'BOTH'] %} + +# docker_container.running returns as soon as the container starts, but on +# first-init docker-entrypoint.sh starts a temporary postgres with +# `listen_addresses=''` to run /docker-entrypoint-initdb.d scripts, then +# shuts it down before exec'ing the real CMD. A default pg_isready check +# (Unix socket) passes during that ephemeral phase and races the shutdown +# with "the database system is shutting down". Checking TCP readiness on +# 127.0.0.1 only succeeds after the final postgres binds the port. +postgres_wait_ready: + cmd.run: + - name: | + for i in $(seq 1 60); do + if docker exec so-postgres pg_isready -h 127.0.0.1 -U postgres -q 2>/dev/null; then + exit 0 + fi + sleep 2 + done + echo "so-postgres did not accept TCP connections within 120s" >&2 + exit 1 + - require: + - docker_container: so-postgres + +# Ensure the shared Telegraf database exists. init-db.sh only runs on a +# fresh data dir, so hosts upgraded onto an existing /nsm/postgres volume +# would otherwise never get so_telegraf. +postgres_create_telegraf_db: + cmd.run: + - name: /usr/sbin/so-telegraf-postgres create_db + - require: + - cmd: postgres_wait_ready + - file: postgres_sbin + +# Provision the shared group role and schema once. Every per-minion role is a +# member of so_telegraf, and each Telegraf connection does SET ROLE so_telegraf +# (via options='-c role=so_telegraf' in the connection string) so tables created +# on first write are owned by the group role and every member can INSERT/SELECT. +postgres_telegraf_group_role: + cmd.run: + - name: /usr/sbin/so-telegraf-postgres group_role + - require: + - cmd: postgres_create_telegraf_db + - file: postgres_sbin + +{% set creds = salt['pillar.get']('telegraf:postgres_creds', {}) %} +{% for mid, entry in creds.items() %} +{% if entry.get('user') and entry.get('pass') %} +{% set u = entry.user %} +{% set p = entry.pass %} + +postgres_telegraf_role_{{ u }}: + cmd.run: + - name: /usr/sbin/so-telegraf-postgres user + - env: + - ROLE_USER: {{ u | tojson }} + - ROLE_PASS: {{ p | tojson }} + - hide_output: True + - require: + - file: postgres_sbin + - cmd: postgres_telegraf_group_role + +{% endif %} +{% endfor %} + +# Reconcile partman retention from pillar. Runs after role/schema setup so +# any partitioned parents Telegraf has already created get their retention +# refreshed whenever postgres.telegraf.retention_days changes. +{% set retention = salt['pillar.get']('postgres:telegraf:retention_days', 14) | int %} +postgres_telegraf_retention_reconcile: + cmd.run: + - name: /usr/sbin/so-telegraf-postgres retention + - env: + - RETENTION_DAYS: {{ retention }} + - require: + - cmd: postgres_telegraf_group_role + - file: postgres_sbin + +{% endif %} + +{% else %} + +{{sls}}_state_not_allowed: + test.fail_without_changes: + - name: {{sls}}_state_not_allowed + +{% endif %} diff --git a/salt/postgres/tools/sbin/so-postgres-backup b/salt/postgres/tools/sbin/so-postgres-backup new file mode 100644 index 000000000..08a73e3a4 --- /dev/null +++ b/salt/postgres/tools/sbin/so-postgres-backup @@ -0,0 +1,73 @@ +#!/bin/bash +# +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +. /usr/sbin/so-common + +# Without pipefail, a pipeline's exit status is gzip's. A failed pg_dumpall would +# otherwise be masked by a successful gzip, silently producing a valid .gz that +# holds a truncated dump. +set -o pipefail + +# Backups contain role password hashes and full chat data; keep them 0600. +umask 0077 + +TODAY=$(date '+%Y_%m_%d') +BACKUPDIR=/nsm/backup +BACKUPFILE="$BACKUPDIR/so-postgres-backup-$TODAY.sql.gz" +TMPFILE="$BACKUPFILE.tmp" +MAXBACKUPS=7 +LOGFILE=/opt/so/log/postgres/backup.log + +log() { + echo "$(date '+%Y-%m-%d %H:%M:%S') $*" >> "$LOGFILE" +} + +mkdir -p "$BACKUPDIR" + +# Remove any temp files left behind by a previously crashed run +rm -f "$BACKUPDIR"/so-postgres-backup-*.sql.gz.tmp + +# Skip if already backed up today +if [ -f "$BACKUPFILE" ]; then + exit 0 +fi + +# Skip if container isn't running +if ! docker ps --format '{{.Names}}' | grep -q '^so-postgres$'; then + exit 0 +fi + +# Always clean up the temp file on exit; the success path clears this trap +# after the atomic rename so the finished backup is not deleted. +trap 'rm -f "$TMPFILE"' EXIT + +# Dump all databases and roles, compress. Write to a temp file so the final +# filename only ever appears for a complete, verified backup. +if ! docker exec so-postgres pg_dumpall -U postgres | gzip > "$TMPFILE"; then + log "ERROR: pg_dumpall/gzip failed; backup aborted" + exit 1 +fi + +# Verify the compressed stream is intact before publishing it +if ! gzip -t "$TMPFILE"; then + log "ERROR: backup failed gzip integrity check; backup aborted" + exit 1 +fi + +# Atomically publish the verified backup +mv "$TMPFILE" "$BACKUPFILE" +trap - EXIT +log "OK: wrote $BACKUPFILE" + +# Retention cleanup (only reached after a successful backup). The glob is +# restricted to finished backups so an in-progress .tmp can never be counted. +NUMBACKUPS=$(find "$BACKUPDIR" -type f -name "so-postgres-backup-*.sql.gz" | wc -l) +while [ "$NUMBACKUPS" -gt "$MAXBACKUPS" ]; do + OLDEST=$(find "$BACKUPDIR" -type f -name "so-postgres-backup-*.sql.gz" -printf '%T+ %p\n' | sort | head -n 1 | awk -F" " '{print $2}') + rm -f "$OLDEST" + NUMBACKUPS=$(find "$BACKUPDIR" -type f -name "so-postgres-backup-*.sql.gz" | wc -l) +done diff --git a/salt/postgres/tools/sbin/so-postgres-manage b/salt/postgres/tools/sbin/so-postgres-manage new file mode 100644 index 000000000..3729d5c0d --- /dev/null +++ b/salt/postgres/tools/sbin/so-postgres-manage @@ -0,0 +1,80 @@ +#!/bin/bash + +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +. /usr/sbin/so-common + +usage() { + echo "Usage: $0 [args]" + echo "" + echo "Supported Operations:" + echo " sql Execute a SQL command, requires: " + echo " sqlfile Execute a SQL file, requires: " + echo " shell Open an interactive psql shell" + echo " dblist List databases" + echo " userlist List database roles" + echo "" + exit 1 +} + +if [ $# -lt 1 ]; then + usage +fi + +# Check for prerequisites +if [ "$(id -u)" -ne 0 ]; then + echo "This script must be run using sudo!" + exit 1 +fi + +COMMAND=$(basename $0) +OP=$1 +shift + +set -eo pipefail + +log() { + echo -e "$(date) | $COMMAND | $@" >&2 +} + +so_psql() { + docker exec so-postgres psql -U postgres -d securityonion "$@" +} + +case "$OP" in + + sql) + [ $# -lt 1 ] && usage + so_psql -c "$1" + ;; + + sqlfile) + [ $# -ne 1 ] && usage + if [ ! -f "$1" ]; then + log "File not found: $1" + exit 1 + fi + docker cp "$1" so-postgres:/tmp/sqlfile.sql + docker exec so-postgres psql -U postgres -d securityonion -f /tmp/sqlfile.sql + docker exec so-postgres rm -f /tmp/sqlfile.sql + ;; + + shell) + docker exec -it so-postgres psql -U postgres -d securityonion + ;; + + dblist) + so_psql -c "\l" + ;; + + userlist) + so_psql -c "\du" + ;; + + *) + usage + ;; +esac diff --git a/salt/postgres/tools/sbin/so-postgres-restart b/salt/postgres/tools/sbin/so-postgres-restart new file mode 100644 index 000000000..8e3e516dd --- /dev/null +++ b/salt/postgres/tools/sbin/so-postgres-restart @@ -0,0 +1,10 @@ +#!/bin/bash + +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +. /usr/sbin/so-common + +/usr/sbin/so-restart postgres $1 diff --git a/salt/postgres/tools/sbin/so-postgres-start b/salt/postgres/tools/sbin/so-postgres-start new file mode 100644 index 000000000..0893eaa2d --- /dev/null +++ b/salt/postgres/tools/sbin/so-postgres-start @@ -0,0 +1,10 @@ +#!/bin/bash + +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +. /usr/sbin/so-common + +/usr/sbin/so-start postgres $1 diff --git a/salt/postgres/tools/sbin/so-postgres-stop b/salt/postgres/tools/sbin/so-postgres-stop new file mode 100644 index 000000000..6fd0d9165 --- /dev/null +++ b/salt/postgres/tools/sbin/so-postgres-stop @@ -0,0 +1,10 @@ +#!/bin/bash + +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +. /usr/sbin/so-common + +/usr/sbin/so-stop postgres $1 diff --git a/salt/postgres/tools/sbin/so-stats-show b/salt/postgres/tools/sbin/so-stats-show new file mode 100644 index 000000000..3cf7a05d8 --- /dev/null +++ b/salt/postgres/tools/sbin/so-stats-show @@ -0,0 +1,157 @@ +#!/bin/bash + +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +# Point-in-time host metrics from the Telegraf Postgres backend. +# Sanity-check tool for verifying metrics are landing before the grid +# dashboards consume them. +# +# Assumes Telegraf's postgresql output is configured with +# tags_as_foreign_keys = true, tags_as_jsonb = true, fields_as_jsonb = true, +# so metric tables are (time, tag_id, fields jsonb) and tag tables are +# (tag_id, tags jsonb). + +. /usr/sbin/so-common + +usage() { + cat <&2 + exit 1 +fi + +so_psql() { + docker exec so-postgres psql -U postgres -d so_telegraf -At -F $'\t' "$@" +} + +if ! docker exec so-postgres psql -U postgres -lqt 2>/dev/null | cut -d\| -f1 | grep -qw so_telegraf; then + echo "Database so_telegraf not found. Is telegraf.output set to POSTGRES or BOTH?" + exit 2 +fi + +table_exists() { + local table="$1" + [ -n "$(so_psql -c "SELECT 1 FROM information_schema.tables WHERE table_schema='${SCHEMA}' AND table_name='${table}' LIMIT 1;")" ] +} + +# Discover hosts from cpu_tag (every minion reports cpu). +if ! table_exists "cpu_tag"; then + echo "${SCHEMA}.cpu_tag not found. Has Telegraf written any rows yet?" + exit 0 +fi + +HOSTS=$(so_psql -c " + SELECT DISTINCT tags->>'host' + FROM \"${SCHEMA}\".cpu_tag + WHERE tags ? 'host' + ORDER BY 1;") + +if [ -z "$HOSTS" ]; then + echo "No hosts found in ${SCHEMA}. Is Telegraf configured to write to Postgres?" + exit 0 +fi + +print_metric() { + so_psql -c "$1" +} + +for host in $HOSTS; do + if ! [[ "$host" =~ $HOST_RE ]]; then + echo "Skipping host with invalid characters in tag value: $host" >&2 + continue + fi + if [ -n "$FILTER_HOST" ] && [ "$host" != "$FILTER_HOST" ]; then + continue + fi + + echo "====================================================================" + echo " Host: $host" + echo "====================================================================" + + if table_exists "cpu"; then + print_metric " + SELECT 'cpu ' AS metric, + to_char(c.time, 'YYYY-MM-DD HH24:MI:SS') AS ts, + round((100 - (c.fields->>'usage_idle')::numeric), 1) || '% used' + FROM \"${SCHEMA}\".cpu c + JOIN \"${SCHEMA}\".cpu_tag t USING (tag_id) + WHERE t.tags->>'host' = '${host}' AND t.tags->>'cpu' = 'cpu-total' + ORDER BY c.time DESC LIMIT 1;" + fi + + if table_exists "mem"; then + print_metric " + SELECT 'memory ' AS metric, + to_char(m.time, 'YYYY-MM-DD HH24:MI:SS') AS ts, + round((m.fields->>'used_percent')::numeric, 1) || '% used (' || + pg_size_pretty((m.fields->>'used')::bigint) || ' of ' || + pg_size_pretty((m.fields->>'total')::bigint) || ')' + FROM \"${SCHEMA}\".mem m + JOIN \"${SCHEMA}\".mem_tag t USING (tag_id) + WHERE t.tags->>'host' = '${host}' + ORDER BY m.time DESC LIMIT 1;" + fi + + if table_exists "disk"; then + print_metric " + SELECT 'disk ' || rpad(t.tags->>'path', 12) AS metric, + to_char(d.time, 'YYYY-MM-DD HH24:MI:SS') AS ts, + round((d.fields->>'used_percent')::numeric, 1) || '% used (' || + pg_size_pretty((d.fields->>'used')::bigint) || ' of ' || + pg_size_pretty((d.fields->>'total')::bigint) || ')' + FROM \"${SCHEMA}\".disk d + JOIN \"${SCHEMA}\".disk_tag t USING (tag_id) + WHERE t.tags->>'host' = '${host}' + AND d.time = (SELECT max(d2.time) + FROM \"${SCHEMA}\".disk d2 + JOIN \"${SCHEMA}\".disk_tag t2 USING (tag_id) + WHERE t2.tags->>'host' = '${host}') + ORDER BY t.tags->>'path';" + fi + + if table_exists "system"; then + print_metric " + SELECT 'load ' AS metric, + to_char(s.time, 'YYYY-MM-DD HH24:MI:SS') AS ts, + (s.fields->>'load1') || ' / ' || + (s.fields->>'load5') || ' / ' || + (s.fields->>'load15') || ' (1/5/15m)' + FROM \"${SCHEMA}\".system s + JOIN \"${SCHEMA}\".system_tag t USING (tag_id) + WHERE t.tags->>'host' = '${host}' + ORDER BY s.time DESC LIMIT 1;" + fi + + echo "" +done diff --git a/salt/postgres/tools/sbin/so-telegraf-postgres b/salt/postgres/tools/sbin/so-telegraf-postgres new file mode 100644 index 000000000..ef7c3f9e6 --- /dev/null +++ b/salt/postgres/tools/sbin/so-telegraf-postgres @@ -0,0 +1,110 @@ +#!/bin/bash +set -e + +# Provision Telegraf state inside the so-postgres container. +# Usage: so-telegraf-postgres +# create_db Ensure the so_telegraf database exists. +# group_role Provision the so_telegraf group role, telegraf/partman schemas, +# pg_partman, pg_cron, and the hourly partman maintenance job. +# user Create or update a per-minion login role granted to so_telegraf. +# Env: ROLE_USER, ROLE_PASS. +# retention Reconcile partman retention on telegraf parents. +# Env: RETENTION_DAYS. + +cmd="${1:?subcommand required}" + +case "$cmd" in + create_db) + if ! docker exec so-postgres psql -U postgres -tAc \ + "SELECT 1 FROM pg_database WHERE datname='so_telegraf'" | grep -q 1; then + docker exec so-postgres psql -v ON_ERROR_STOP=1 -U postgres \ + -c "CREATE DATABASE so_telegraf" + fi + ;; + + group_role) + docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -U postgres -d so_telegraf <<'EOSQL' +DO $$ +BEGIN + IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = 'so_telegraf') THEN + CREATE ROLE so_telegraf NOLOGIN; + END IF; +END +$$; +GRANT CONNECT ON DATABASE so_telegraf TO so_telegraf; +CREATE SCHEMA IF NOT EXISTS telegraf AUTHORIZATION so_telegraf; +GRANT USAGE, CREATE ON SCHEMA telegraf TO so_telegraf; +CREATE SCHEMA IF NOT EXISTS partman; +CREATE EXTENSION IF NOT EXISTS pg_partman SCHEMA partman; +CREATE EXTENSION IF NOT EXISTS pg_cron; +-- Telegraf (running as so_telegraf) calls partman.create_parent() +-- on first write of each metric, which needs USAGE on the partman +-- schema, EXECUTE on its functions/procedures, and write access to +-- partman.part_config so it can register new partitioned parents. +GRANT USAGE, CREATE ON SCHEMA partman TO so_telegraf; +GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA partman TO so_telegraf; +GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA partman TO so_telegraf; +GRANT EXECUTE ON ALL PROCEDURES IN SCHEMA partman TO so_telegraf; +-- partman creates per-parent template tables (partman.template_*) at +-- runtime; default privileges extend DML/sequence access to them. +ALTER DEFAULT PRIVILEGES IN SCHEMA partman + GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO so_telegraf; +ALTER DEFAULT PRIVILEGES IN SCHEMA partman + GRANT USAGE, SELECT, UPDATE ON SEQUENCES TO so_telegraf; +-- Hourly partman maintenance. cron.schedule is idempotent by jobname. +SELECT cron.schedule( + 'telegraf-partman-maintenance', + '17 * * * *', + 'CALL partman.run_maintenance_proc()' +); +EOSQL + ;; + + user) + : "${ROLE_USER:?ROLE_USER is required}" + : "${ROLE_PASS:?ROLE_PASS is required}" + # psql does not substitute :vars inside dollar-quoted strings, so the + # conditional CREATE/ALTER is built outside any DO block and dispatched + # with \gexec. format() handles identifier/literal quoting. + docker exec -i so-postgres psql \ + -v ON_ERROR_STOP=1 \ + -v role_user="$ROLE_USER" \ + -v role_pass="$ROLE_PASS" \ + -U postgres -d so_telegraf <<'EOSQL' +SELECT format( + CASE WHEN EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = :'role_user') + THEN 'ALTER ROLE %I WITH LOGIN PASSWORD %L' + ELSE 'CREATE ROLE %I WITH LOGIN PASSWORD %L' + END, + :'role_user', + :'role_pass' +) \gexec +GRANT CONNECT ON DATABASE so_telegraf TO :"role_user"; +GRANT so_telegraf TO :"role_user"; +EOSQL + ;; + + retention) + : "${RETENTION_DAYS:?RETENTION_DAYS is required}" + # \gset + \if guards against a missing pg_partman without using a DO + # block (psql :var substitution doesn't reach into dollar-quoted code). + docker exec -i so-postgres psql \ + -v ON_ERROR_STOP=1 \ + -v retention_days="$RETENTION_DAYS" \ + -U postgres -d so_telegraf <<'EOSQL' +SELECT CASE WHEN EXISTS (SELECT 1 FROM pg_catalog.pg_extension WHERE extname = 'pg_partman') + THEN 'true' ELSE 'false' END AS has_partman \gset +\if :has_partman +UPDATE partman.part_config +SET retention = :'retention_days' || ' days', + retention_keep_table = false +WHERE parent_table LIKE 'telegraf.%'; +\endif +EOSQL + ;; + + *) + echo "Unknown subcommand: $cmd" >&2 + exit 1 + ;; +esac diff --git a/salt/reactor/check_hypervisor.sls b/salt/reactor/check_hypervisor.sls index 91b7c0c02..da81e0d5f 100644 --- a/salt/reactor/check_hypervisor.sls +++ b/salt/reactor/check_hypervisor.sls @@ -3,12 +3,15 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -{% if data['id'].endswith('_hypervisor') and data['result'] == True %} +{% set hid = data['id'] %} +{% if hid|regex_match('^([A-Za-z0-9._-]{1,253})$') + and hid.endswith('_hypervisor') + and data['result'] == True %} {% if data['act'] == 'accept' %} check_and_trigger: runner.setup_hypervisor.setup_environment: - - minion_id: {{ data['id'] }} + - minion_id: {{ hid }} {% endif %} {% if data['act'] == 'delete' %} @@ -17,8 +20,7 @@ delete_hypervisor: - args: - mods: orch.delete_hypervisor - pillar: - minion_id: {{ data['id'] }} + minion_id: {{ hid }} {% endif %} {% endif %} - diff --git a/salt/reactor/createEmptyPillar.sls b/salt/reactor/createEmptyPillar.sls index c6c655bab..2076b53bd 100644 --- a/salt/reactor/createEmptyPillar.sls +++ b/salt/reactor/createEmptyPillar.sls @@ -1,7 +1,7 @@ #!py # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. @@ -9,30 +9,42 @@ import logging import os import pwd import grp +import re + +log = logging.getLogger(__name__) + +PILLAR_ROOT = '/opt/so/saltstack/local/pillar/minions/' +_VMNAME_RE = re.compile(r'^[A-Za-z0-9._-]{1,253}$') + def run(): - vm_name = data['kwargs']['name'] - logging.error("createEmptyPillar reactor: vm_name: %s" % vm_name) - pillar_root = '/opt/so/saltstack/local/pillar/minions/' + vm_name = data.get('kwargs', {}).get('name', '') + if not _VMNAME_RE.match(str(vm_name)): + log.error("createEmptyPillar reactor: refusing unsafe vm_name=%r", vm_name) + return {} + + log.info("createEmptyPillar reactor: vm_name: %s", vm_name) pillar_files = ['adv_' + vm_name + '.sls', vm_name + '.sls'] try: - # Get socore user and group IDs socore_uid = pwd.getpwnam('socore').pw_uid socore_gid = grp.getgrnam('socore').gr_gid + pillar_root_real = os.path.realpath(PILLAR_ROOT) for f in pillar_files: - full_path = pillar_root + f - if not os.path.exists(full_path): - # Create empty file - os.mknod(full_path) - # Set ownership to socore:socore - os.chown(full_path, socore_uid, socore_gid) - # Set mode to 644 (rw-r--r--) - os.chmod(full_path, 0o640) - logging.error("createEmptyPillar reactor: created %s with socore:socore ownership and mode 644" % f) + full_path = os.path.join(PILLAR_ROOT, f) + resolved = os.path.realpath(full_path) + if os.path.dirname(resolved) != pillar_root_real: + log.error("createEmptyPillar reactor: refusing path outside pillar root: %s", resolved) + continue + if os.path.exists(resolved): + continue + os.mknod(resolved) + os.chown(resolved, socore_uid, socore_gid) + os.chmod(resolved, 0o640) + log.info("createEmptyPillar reactor: created %s with socore:socore ownership and mode 0640", f) except (KeyError, OSError) as e: - logging.error("createEmptyPillar reactor: Error setting ownership/permissions: %s" % str(e)) + log.error("createEmptyPillar reactor: Error setting ownership/permissions: %s", e) return {} diff --git a/salt/reactor/deleteKey.sls b/salt/reactor/deleteKey.sls index 4d522a4b5..a57d6370c 100644 --- a/salt/reactor/deleteKey.sls +++ b/salt/reactor/deleteKey.sls @@ -1,18 +1,40 @@ +#!py + # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one # or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -remove_key: - wheel.key.delete: - - args: - - match: {{ data['name'] }} +import logging +import re -{{ data['name'] }}_pillar_clean: - runner.state.orchestrate: - - args: - - mods: orch.vm_pillar_clean - - pillar: - vm_name: {{ data['name'] }} +log = logging.getLogger(__name__) -{% do salt.log.info('deleteKey reactor: deleted minion key: %s' % data['name']) %} +_VMNAME_RE = re.compile(r'^[A-Za-z0-9._-]{1,253}$') + + +def run(): + name = data.get('name', '') + if not _VMNAME_RE.match(str(name)): + log.error("deleteKey reactor: refusing unsafe name=%r", name) + return {} + + log.info("deleteKey reactor: deleted minion key: %s", name) + + return { + 'remove_key': { + 'wheel.key.delete': [ + {'args': [ + {'match': name}, + ]}, + ], + }, + '%s_pillar_clean' % name: { + 'runner.state.orchestrate': [ + {'args': [ + {'mods': 'orch.vm_pillar_clean'}, + {'pillar': {'vm_name': name}}, + ]}, + ], + }, + } diff --git a/salt/reactor/sominion_setup.sls b/salt/reactor/sominion_setup.sls index 64b3666f4..a72348cc8 100644 --- a/salt/reactor/sominion_setup.sls +++ b/salt/reactor/sominion_setup.sls @@ -6,39 +6,74 @@ # Elastic License 2.0. import logging -from subprocess import call -import yaml +import os +import re +import shlex +import subprocess log = logging.getLogger(__name__) +SO_MINION = '/usr/sbin/so-minion' + +_NODETYPE_RE = re.compile(r'^[A-Z][A-Z0-9_]{0,31}$') +_MINIONID_RE = re.compile(r'^[A-Za-z0-9._-]{1,253}$') +_HOSTPART_RE = re.compile(r'^[A-Za-z0-9._-]{1,253}$') +_IPV4_RE = re.compile( + r'^(?:(?:25[0-5]|2[0-4]\d|[01]?\d?\d)\.){3}' + r'(?:25[0-5]|2[0-4]\d|[01]?\d?\d)$' +) +_HEAP_RE = re.compile(r'^\d{1,6}[kKmMgG]?$') + + +def _check(name, value, pattern): + s = str(value) + if not pattern.match(s): + raise ValueError("sominion_setup_reactor: refusing unsafe %s=%r" % (name, value)) + return s + + def run(): log.info('sominion_setup_reactor: Running') minionid = data['id'] DATA = data['data'] - hv_name = DATA['HYPERVISOR_HOST'] log.info('sominion_setup_reactor: DATA: %s' % DATA) - # Build the base command - cmd = "NODETYPE=" + DATA['NODETYPE'] + " /usr/sbin/so-minion -o=addVM -m=" + minionid + " -n=" + DATA['MNIC'] + " -i=" + DATA['MAINIP'] + " -c=" + str(DATA['CPUCORES']) + " -d='" + DATA['NODE_DESCRIPTION'] + "'" - - # Add optional arguments only if they exist in DATA + nodetype = _check('NODETYPE', DATA['NODETYPE'], _NODETYPE_RE) + + argv = [ + SO_MINION, + '-o=addVM', + '-m=' + _check('minionid', minionid, _MINIONID_RE), + '-n=' + _check('MNIC', DATA['MNIC'], _HOSTPART_RE), + '-i=' + _check('MAINIP', DATA['MAINIP'], _IPV4_RE), + '-c=' + str(int(DATA['CPUCORES'])), + '-d=' + str(DATA['NODE_DESCRIPTION']), + ] + if 'CORECOUNT' in DATA: - cmd += " -C=" + str(DATA['CORECOUNT']) - + argv.append('-C=' + str(int(DATA['CORECOUNT']))) + if 'INTERFACE' in DATA: - cmd += " -a=" + DATA['INTERFACE'] - + argv.append('-a=' + _check('INTERFACE', DATA['INTERFACE'], _HOSTPART_RE)) + if 'ES_HEAP_SIZE' in DATA: - cmd += " -e=" + DATA['ES_HEAP_SIZE'] - + argv.append('-e=' + _check('ES_HEAP_SIZE', DATA['ES_HEAP_SIZE'], _HEAP_RE)) + if 'LS_HEAP_SIZE' in DATA: - cmd += " -l=" + DATA['LS_HEAP_SIZE'] + argv.append('-l=' + _check('LS_HEAP_SIZE', DATA['LS_HEAP_SIZE'], _HEAP_RE)) if 'LSHOSTNAME' in DATA: - cmd += " -L=" + DATA['LSHOSTNAME'] - - log.info('sominion_setup_reactor: Command: %s' % cmd) - rc = call(cmd, shell=True) + argv.append('-L=' + _check('LSHOSTNAME', DATA['LSHOSTNAME'], _HOSTPART_RE)) + + env = os.environ.copy() + env['NODETYPE'] = nodetype + + log.info( + 'sominion_setup_reactor: argv: %s (NODETYPE=%s)', + ' '.join(shlex.quote(a) for a in argv), + shlex.quote(nodetype), + ) + rc = subprocess.call(argv, shell=False, env=env) log.info('sominion_setup_reactor: rc: %s' % rc) diff --git a/salt/salt/cloud/cloud.profiles.d/socloud.conf.jinja b/salt/salt/cloud/cloud.profiles.d/socloud.conf.jinja index f8e5d5555..44bef4108 100644 --- a/salt/salt/cloud/cloud.profiles.d/socloud.conf.jinja +++ b/salt/salt/cloud/cloud.profiles.d/socloud.conf.jinja @@ -27,6 +27,7 @@ sool9_{{host}}: log_file: /opt/so/log/salt/minion grains: hypervisor_host: {{host ~ "_" ~ role}} + sosmodel: HVGUEST preflight_cmds: - | {%- set hostnames = [MANAGERHOSTNAME] %} diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 0bde8f20e..cc80758fc 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -2687,4 +2687,5 @@ soc: lowBalanceColorAlert: 500000 enabled: true adapter: SOAI + charsPerTokenEstimate: 4 diff --git a/salt/soc/files/soc/sigma_so_pipeline.yaml b/salt/soc/files/soc/sigma_so_pipeline.yaml index 4462bde42..b5bc96dc4 100644 --- a/salt/soc/files/soc/sigma_so_pipeline.yaml +++ b/salt/soc/files/soc/sigma_so_pipeline.yaml @@ -117,6 +117,121 @@ transformations: - type: logsource product: linux service: auth + # Maps M365 audit rules to Elastic Agent O365 integration logs + - id: m365_audit_field_mappings + type: field_name_mapping + mapping: + Operation: event.action + ResultStatus: event.outcome + ApplicationId: o365.audit.ApplicationId + ObjectId: o365.audit.ObjectId + RequestType: o365.audit.RequestType + rule_conditions: + - type: logsource + product: m365 + service: audit + - id: m365_audit_add-fields + type: add_condition + conditions: + event.dataset: 'o365.audit' + event.module: 'o365' + rule_conditions: + - type: logsource + product: m365 + service: audit + # Maps M365 exchange rules to Elastic Agent O365 integration logs + - id: m365_exchange_field_mappings + type: field_name_mapping + mapping: + eventSource: event.provider + eventName: event.action + status: event.outcome + rule_conditions: + - type: logsource + product: m365 + service: exchange + - id: m365_exchange_add-fields + type: add_condition + conditions: + event.dataset: 'o365.audit' + event.module: 'o365' + rule_conditions: + - type: logsource + product: m365 + service: exchange + # Maps M365 threat_management rules to Elastic Agent O365 integration logs + - id: m365_threat_management_field_mappings + type: field_name_mapping + mapping: + eventSource: event.provider + eventName: event.action + status: event.outcome + rule_conditions: + - type: logsource + product: m365 + service: threat_management + - id: m365_threat_management_add-fields + type: add_condition + conditions: + event.dataset: 'o365.audit' + event.module: 'o365' + rule_conditions: + - type: logsource + product: m365 + service: threat_management + # Maps M365 threat_detection rules to Elastic Agent O365 integration logs + - id: m365_threat_detection_field_mappings + type: field_name_mapping + mapping: + eventSource: event.provider + eventName: event.action + status: event.outcome + rule_conditions: + - type: logsource + product: m365 + service: threat_detection + - id: m365_threat_detection_add-fields + type: add_condition + conditions: + event.dataset: 'o365.audit' + event.module: 'o365' + rule_conditions: + - type: logsource + product: m365 + service: threat_detection + # Maps FortiGate event rules to Elastic Agent Fortinet integration logs + - id: fortigate_event_field_mappings + type: field_name_mapping + mapping: + action: fortinet.firewall.action + cfgpath: fortinet.firewall.cfgpath + cfgobj: fortinet.firewall.cfgobj + cfgattr: fortinet.firewall.cfgattr + devname: observer.name + devid: observer.serial_number + logid: event.code + type: fortinet.firewall.type + subtype: fortinet.firewall.subtype + level: log.level + vd: fortinet.firewall.vd + logdesc: fortinet.firewall.desc + user: user.name + ui: fortinet.firewall.ui + cfgtid: fortinet.firewall.cfgtid + msg: message + rule_conditions: + - type: logsource + product: fortigate + service: event + - id: fortigate_event_add-fields + type: add_condition + conditions: + event.dataset: 'fortinet_fortigate.log' + event.module: 'fortinet_fortigate' + rule_conditions: + - type: logsource + product: fortigate + service: event # event.code should always be a string - id: convert_event_code_to_string type: convert_type @@ -126,15 +241,36 @@ transformations: fields: - event.code # Maps process_creation rules to endpoint process creation logs - # This is an OS-agnostic mapping, to account for logs that don't specify source OS - id: endpoint_process_create_windows_add-fields type: add_condition conditions: event.category: 'process' event.type: 'start' + host.os.type: 'windows' rule_conditions: - type: logsource category: process_creation + product: windows + - id: endpoint_process_create_macos_add-fields + type: add_condition + conditions: + event.category: 'process' + event.type: 'start' + host.os.type: 'macos' + rule_conditions: + - type: logsource + category: process_creation + product: macos + - id: endpoint_process_create_linux_add-fields + type: add_condition + conditions: + event.category: 'process' + event.type: 'start' + host.os.type: 'linux' + rule_conditions: + - type: logsource + category: process_creation + product: linux # Maps file_event rules to endpoint file creation logs # This is an OS-agnostic mapping, to account for logs that don't specify source OS - id: endpoint_file_create_add-fields diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index c5f96894d..6a2f79629 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -3,6 +3,7 @@ soc: description: Enables or disables SOC. WARNING - Disabling this setting is unsupported and will cause the grid to malfunction. Re-enabling this setting is a manual effort via SSH. forcedType: bool advanced: True + readonly: True telemetryEnabled: title: SOC Telemetry description: When this setting is enabled and the grid is not in airgap mode, SOC will provide feature usage data to the Security Onion development team via Google Analytics. This data helps Security Onion developers determine which product features are being used and can also provide insight into improving the user interface. When changing this setting, wait for the grid to fully synchronize and then perform a hard browser refresh on SOC, to force the browser cache to update and reflect the new setting. @@ -761,7 +762,7 @@ soc: required: True - field: origin label: Country of Origin for the Model Training - required: false + required: False - field: contextLimitSmall label: Context Limit (Small) forcedType: int @@ -779,6 +780,10 @@ soc: - field: enabled label: Enabled forcedType: bool + - field: charsPerTokenEstimate + label: Characters per Token Estimate + forcedType: float + required: False apiTimeoutMs: description: Duration (in milliseconds) to wait for a response from the SOC server API before giving up and showing an error on the SOC UI. global: True @@ -886,12 +891,16 @@ soc: suricata: description: The template used when creating a new Suricata detection. [publicId] will be replaced with an unused Public Id. multiline: True + forcedType: string strelka: description: The template used when creating a new Strelka detection. multiline: True + forcedType: string elastalert: description: The template used when creating a new ElastAlert detection. [publicId] will be replaced with an unused Public Id. multiline: True + forcedType: string + grid: maxUploadSize: description: The maximum number of bytes for an uploaded PCAP import file. diff --git a/salt/strelka/defaults.yaml b/salt/strelka/defaults.yaml index 4d69bf53b..302a45af4 100644 --- a/salt/strelka/defaults.yaml +++ b/salt/strelka/defaults.yaml @@ -261,7 +261,7 @@ strelka: priority: 5 options: limit: 1000 - 'ScanLNK': + 'ScanLnk': - positive: flavors: - 'lnk_file' diff --git a/salt/strelka/filecheck/filecheck b/salt/strelka/filecheck/filecheck index 758248083..35b47ce71 100644 --- a/salt/strelka/filecheck/filecheck +++ b/salt/strelka/filecheck/filecheck @@ -15,7 +15,7 @@ from watchdog.observers import Observer from watchdog.events import FileSystemEventHandler with open("/opt/so/conf/strelka/filecheck.yaml", "r") as ymlfile: - cfg = yaml.load(ymlfile, Loader=yaml.Loader) + cfg = yaml.safe_load(ymlfile) extract_path = cfg["filecheck"]["extract_path"] historypath = cfg["filecheck"]["historypath"] diff --git a/salt/strelka/soc_strelka.yaml b/salt/strelka/soc_strelka.yaml index 0066bd6c3..65ff950b8 100644 --- a/salt/strelka/soc_strelka.yaml +++ b/salt/strelka/soc_strelka.yaml @@ -99,7 +99,7 @@ strelka: 'ScanJpeg': *scannerOptions 'ScanJson': *scannerOptions 'ScanLibarchive': *scannerOptions - 'ScanLNK': *scannerOptions + 'ScanLnk': *scannerOptions 'ScanLsb': *scannerOptions 'ScanLzma': *scannerOptions 'ScanMacho': *scannerOptions diff --git a/salt/suricata/map.jinja b/salt/suricata/map.jinja index 944e0e34d..f7059b293 100644 --- a/salt/suricata/map.jinja +++ b/salt/suricata/map.jinja @@ -33,7 +33,7 @@ {% do SURICATAMERGED.config.outputs['pcap-log'].update({'conditional': SURICATAMERGED.pcap.conditional}) %} {% do SURICATAMERGED.config.outputs['pcap-log'].update({'dir': SURICATAMERGED.pcap.dir}) %} {# multiply maxsize by 1000 since it is saved in GB, i.e. 52 = 52000MB. filesize is also saved in MB and we strip the MB and convert to int #} -{% set maxfiles = (SURICATAMERGED.pcap.maxsize * 1000 / (SURICATAMERGED.pcap.filesize[:-2] | int) / SURICATAMERGED.config['af-packet'].threads | int) | round | int %} +{% set maxfiles = ([1, (SURICATAMERGED.pcap.maxsize * 1000 / (SURICATAMERGED.pcap.filesize[:-2] | int) / SURICATAMERGED.config['af-packet'].threads | int) | round(0, 'ceil') | int] | max) %} {% do SURICATAMERGED.config.outputs['pcap-log'].update({'max-files': maxfiles}) %} {% endif %} diff --git a/salt/suricata/soc_suricata.yaml b/salt/suricata/soc_suricata.yaml index c85b876a9..ce6b7d008 100644 --- a/salt/suricata/soc_suricata.yaml +++ b/salt/suricata/soc_suricata.yaml @@ -64,8 +64,10 @@ suricata: helpLink: suricata conditional: description: Set to "all" to record PCAP for all flows. Set to "alerts" to only record PCAP for Suricata alerts. Set to "tag" to only record PCAP for tagged rules. - regex: ^(all|alerts|tag)$ - regexFailureMessage: You must enter either all, alert or tag. + options: + - all + - alerts + - tag helpLink: suricata dir: description: Parent directory to store PCAP. @@ -83,7 +85,9 @@ suricata: advanced: True cluster-type: advanced: True - regex: ^(cluster_flow|cluster_qm)$ + options: + - cluster_flow + - cluster_qm defrag: description: Enable defragmentation of IP packets before processing. forcedType: bool diff --git a/salt/telegraf/defaults.yaml b/salt/telegraf/defaults.yaml index ef6c2bc77..24f58b157 100644 --- a/salt/telegraf/defaults.yaml +++ b/salt/telegraf/defaults.yaml @@ -1,5 +1,6 @@ telegraf: enabled: False + output: INFLUXDB config: interval: '30s' metric_batch_size: 1000 diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf index aafcf6d77..02d969ff3 100644 --- a/salt/telegraf/etc/telegraf.conf +++ b/salt/telegraf/etc/telegraf.conf @@ -8,6 +8,14 @@ {%- set ZEEK_ENABLED = salt['pillar.get']('zeek:enabled', True) %} {%- set MDENGINE = GLOBALS.md_engine %} {%- set LOGSTASH_ENABLED = LOGSTASH_MERGED.enabled %} +{%- set TG_OUT = TELEGRAFMERGED.output | upper %} +{%- set PG_HOST = GLOBALS.manager_ip %} +{#- Per-minion telegraf creds live in the grid-wide telegraf/creds.sls pillar, + written by /usr/sbin/so-telegraf-cred on the manager. Each minion looks up + its own entry by grains.id. #} +{%- set PG_ENTRY = salt['pillar.get']('telegraf:postgres_creds:' ~ grains.id, {}) %} +{%- set PG_USER = PG_ENTRY.get('user', '') %} +{%- set PG_PASS = PG_ENTRY.get('pass', '') %} # Global tags can be specified here in key="value" format. [global_tags] role = "{{ GLOBALS.role.split('-') | last }}" @@ -72,6 +80,7 @@ # OUTPUT PLUGINS # ############################################################################### +{%- if TG_OUT in ['INFLUXDB', 'BOTH'] %} # Configuration for sending metrics to InfluxDB [[outputs.influxdb_v2]] urls = ["https://{{ INFLUXDBHOST }}:8086"] @@ -85,6 +94,41 @@ tls_key = "/etc/telegraf/telegraf.key" ## Use TLS but skip chain & host verification # insecure_skip_verify = false +{%- endif %} + +{%- if TG_OUT in ['POSTGRES', 'BOTH'] and PG_USER and PG_PASS %} +# Configuration for sending metrics to PostgreSQL. +# options='-c role=so_telegraf' makes every connection SET ROLE to the shared +# group role so tables created on first write are owned by so_telegraf, and +# all per-minion members can INSERT/SELECT them via role inheritance. +# fields_as_jsonb/tags_as_jsonb keep metric tables at a fixed column count so +# high-cardinality inputs (docker, procstat, kafka) don't blow past the +# Postgres 1600-column-per-table limit. +[[outputs.postgresql]] + connection = "host={{ PG_HOST }} port=5432 user={{ PG_USER }} password={{ PG_PASS }} dbname=so_telegraf sslmode=verify-full sslrootcert=/etc/telegraf/ca.crt options='-c role=so_telegraf'" + schema = "telegraf" + tags_as_foreign_keys = true + tags_as_jsonb = true + fields_as_jsonb = true + # Every metric table is a daily time-range partitioned parent managed by + # pg_partman. Retention drops old partitions instead of row-by-row DELETEs. + {% raw %} + # pg_partman 5.x requires the control column (time) to be NOT NULL, so + # ALTER it before create_parent(). And create_parent() splits + # p_parent_table on '.' to look up raw identifiers, so the literal must + # be 'schema.name' (not '"schema"."name"' as .table|quoteLiteral emits). + # IF NOT EXISTS keeps the three templates idempotent so a Telegraf + # restart after any DB-side surgery re-runs them safely. + create_templates = [ + '''CREATE TABLE IF NOT EXISTS {{ .table }} ({{ .columns }}) PARTITION BY RANGE ("time")''', + '''ALTER TABLE {{ .table }} ALTER COLUMN "time" SET NOT NULL''', + '''SELECT partman.create_parent(p_parent_table := {{ printf "%s.%s" .table.Schema .table.Name | quoteLiteral }}, p_control := 'time', p_type := 'range', p_interval := '1 day', p_premake := 3) WHERE NOT EXISTS (SELECT 1 FROM partman.part_config WHERE parent_table = {{ printf "%s.%s" .table.Schema .table.Name | quoteLiteral }})''' + ] + tag_table_create_templates = [ + '''CREATE TABLE IF NOT EXISTS {{ .table }} ({{ .columns }}, PRIMARY KEY (tag_id))''' + ] + {% endraw %} +{%- endif %} ############################################################################### # PROCESSOR PLUGINS # diff --git a/salt/telegraf/soc_telegraf.yaml b/salt/telegraf/soc_telegraf.yaml index 40ae7fed8..4b9a2e3d1 100644 --- a/salt/telegraf/soc_telegraf.yaml +++ b/salt/telegraf/soc_telegraf.yaml @@ -4,6 +4,15 @@ telegraf: forcedType: bool advanced: True helpLink: influxdb + output: + description: Selects the backend(s) Telegraf writes metrics to. INFLUXDB keeps the current behavior; POSTGRES writes to the grid's Postgres instance; BOTH dual-writes for migration validation. + options: + - INFLUXDB + - POSTGRES + - BOTH + global: True + advanced: True + helpLink: influxdb config: interval: description: Data collection interval. diff --git a/salt/top.sls b/salt/top.sls index c7c6aa65d..cf743edd1 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -68,6 +68,7 @@ base: - backup.config_backup - nginx - influxdb + - postgres - soc - kratos - hydra @@ -95,6 +96,7 @@ base: - backup.config_backup - nginx - influxdb + - postgres - soc - kratos - hydra @@ -117,12 +119,13 @@ base: - kafka - pcap.cleanup - '*_manager or *_managerhype and G@saltversion:{{saltversion}} and not I@node_data:False': + '*_manager and G@saltversion:{{saltversion}} and not I@node_data:False': - match: compound - salt.master - registry - nginx - influxdb + - postgres - strelka.manager - soc - kratos @@ -143,6 +146,32 @@ base: - stig - kafka + '*_managerhype and G@saltversion:{{saltversion}} and not I@node_data:False': + - match: compound + - salt.master + - registry + - nginx + - influxdb + - postgres + - strelka.manager + - soc + - kratos + - hydra + - firewall + - manager + - sensoroni + - telegraf + - backup.config_backup + - elasticsearch + - logstash + - redis + - elastic-fleet-package-registry + - kibana + - elastalert + - utility + - elasticfleet + - kafka + '*_managerhype and I@features:vrt and G@saltversion:{{saltversion}}': - match: compound - manager.hypervisor @@ -153,6 +182,7 @@ base: - registry - nginx - influxdb + - postgres - strelka.manager - soc - kratos @@ -181,6 +211,7 @@ base: - manager - nginx - influxdb + - postgres - strelka.manager - soc - kratos @@ -281,7 +312,6 @@ base: - libvirt - libvirt.images - elasticfleet.install_agent_grid - - stig '*_desktop and G@saltversion:{{saltversion}}': - sensoroni diff --git a/salt/vars/eval.map.jinja b/salt/vars/eval.map.jinja index 3c2e66a97..3cba33797 100644 --- a/salt/vars/eval.map.jinja +++ b/salt/vars/eval.map.jinja @@ -1,4 +1,5 @@ {% from 'vars/elasticsearch.map.jinja' import ELASTICSEARCH_GLOBALS %} +{% from 'vars/postgres.map.jinja' import POSTGRES_GLOBALS %} {% from 'vars/sensor.map.jinja' import SENSOR_GLOBALS %} {% set ROLE_GLOBALS = {} %} @@ -6,6 +7,7 @@ {% set EVAL_GLOBALS = [ ELASTICSEARCH_GLOBALS, + POSTGRES_GLOBALS, SENSOR_GLOBALS ] %} diff --git a/salt/vars/import.map.jinja b/salt/vars/import.map.jinja index f9dfa0c25..8dea3ad7d 100644 --- a/salt/vars/import.map.jinja +++ b/salt/vars/import.map.jinja @@ -1,4 +1,5 @@ {% from 'vars/elasticsearch.map.jinja' import ELASTICSEARCH_GLOBALS %} +{% from 'vars/postgres.map.jinja' import POSTGRES_GLOBALS %} {% from 'vars/sensor.map.jinja' import SENSOR_GLOBALS %} {% set ROLE_GLOBALS = {} %} @@ -6,6 +7,7 @@ {% set IMPORT_GLOBALS = [ ELASTICSEARCH_GLOBALS, + POSTGRES_GLOBALS, SENSOR_GLOBALS ] %} diff --git a/salt/vars/manager.map.jinja b/salt/vars/manager.map.jinja index c6b348341..009dd5607 100644 --- a/salt/vars/manager.map.jinja +++ b/salt/vars/manager.map.jinja @@ -1,12 +1,14 @@ {% from 'vars/elasticsearch.map.jinja' import ELASTICSEARCH_GLOBALS %} {% from 'vars/logstash.map.jinja' import LOGSTASH_GLOBALS %} +{% from 'vars/postgres.map.jinja' import POSTGRES_GLOBALS %} {% set ROLE_GLOBALS = {} %} {% set MANAGER_GLOBALS = [ ELASTICSEARCH_GLOBALS, - LOGSTASH_GLOBALS + LOGSTASH_GLOBALS, + POSTGRES_GLOBALS ] %} diff --git a/salt/vars/managersearch.map.jinja b/salt/vars/managersearch.map.jinja index c2a3d9628..369efe5a4 100644 --- a/salt/vars/managersearch.map.jinja +++ b/salt/vars/managersearch.map.jinja @@ -1,12 +1,14 @@ {% from 'vars/elasticsearch.map.jinja' import ELASTICSEARCH_GLOBALS %} {% from 'vars/logstash.map.jinja' import LOGSTASH_GLOBALS %} +{% from 'vars/postgres.map.jinja' import POSTGRES_GLOBALS %} {% set ROLE_GLOBALS = {} %} {% set MANAGERSEARCH_GLOBALS = [ ELASTICSEARCH_GLOBALS, - LOGSTASH_GLOBALS + LOGSTASH_GLOBALS, + POSTGRES_GLOBALS ] %} diff --git a/salt/vars/postgres.map.jinja b/salt/vars/postgres.map.jinja new file mode 100644 index 000000000..ce65d2d1f --- /dev/null +++ b/salt/vars/postgres.map.jinja @@ -0,0 +1,16 @@ +{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one + or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at + https://securityonion.net/license; you may not use this file except in compliance with the + Elastic License 2.0. #} + +{% import 'vars/init.map.jinja' as INIT %} + +{% + set POSTGRES_GLOBALS = { + 'postgres': {} + } +%} + +{% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/postgres/auth.sls') %} +{% do POSTGRES_GLOBALS.postgres.update({'auth': INIT.PILLAR.postgres.auth}) %} +{% endif %} diff --git a/salt/vars/standalone.map.jinja b/salt/vars/standalone.map.jinja index 0e49a327d..6488eb998 100644 --- a/salt/vars/standalone.map.jinja +++ b/salt/vars/standalone.map.jinja @@ -1,5 +1,6 @@ {% from 'vars/elasticsearch.map.jinja' import ELASTICSEARCH_GLOBALS %} {% from 'vars/logstash.map.jinja' import LOGSTASH_GLOBALS %} +{% from 'vars/postgres.map.jinja' import POSTGRES_GLOBALS %} {% from 'vars/sensor.map.jinja' import SENSOR_GLOBALS %} {% set ROLE_GLOBALS = {} %} @@ -8,6 +9,7 @@ [ ELASTICSEARCH_GLOBALS, LOGSTASH_GLOBALS, + POSTGRES_GLOBALS, SENSOR_GLOBALS ] %} diff --git a/setup/so-functions b/setup/so-functions index bf95ea9d8..c94b8eee7 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -202,10 +202,10 @@ check_service_status() { systemctl status $service_name > /dev/null 2>&1 local status=$? if [ $status -gt 0 ]; then - info " $service_name is not running" + info "$service_name is not running" return 1; else - info " $service_name is running" + info "$service_name is running" return 0; fi @@ -745,6 +745,56 @@ configure_network_sensor() { return $err } +configure_management_bond() { + local bond_name="bond1" + local bond_mode=${MBOND_MODE:-active-backup} + + info "Setting up $bond_name management interface with mode $bond_mode" + + if [[ ${#MBNICS[@]} -eq 0 ]]; then + error "[ERROR] No management bond NICs were selected." + fail_setup + fi + + nmcli -t -f NAME con show | grep -Fxq "$bond_name" + local found_int=$? + + if [[ $found_int != 0 ]]; then + nmcli con add type bond ifname "$bond_name" con-name "$bond_name" mode "$bond_mode" -- \ + ipv6.method ignore \ + connection.autoconnect yes >> "$setup_log" 2>&1 + else + nmcli con mod "$bond_name" \ + bond.options "mode=$bond_mode" \ + ipv6.method ignore \ + connection.autoconnect yes >> "$setup_log" 2>&1 + fi + + local err=0 + for MBNIC in "${MBNICS[@]}"; do + local slave_name="$bond_name-slave-$MBNIC" + + nmcli -t -f NAME con show | grep -Fxq "$slave_name" + found_int=$? + + if [[ $found_int != 0 ]]; then + nmcli con add type ethernet ifname "$MBNIC" con-name "$slave_name" master "$bond_name" -- \ + connection.autoconnect yes >> "$setup_log" 2>&1 + else + nmcli con mod "$slave_name" \ + connection.master "$bond_name" \ + connection.slave-type bond \ + connection.autoconnect yes >> "$setup_log" 2>&1 + fi + + nmcli con up "$slave_name" >> "$setup_log" 2>&1 + local ret=$? + [[ $ret -eq 0 ]] || err=$ret + done + + return $err +} + configure_hyper_bridge() { info "Setting up hypervisor bridge" info "Checking $MNIC ipv4.method is auto or manual" @@ -821,6 +871,7 @@ create_manager_pillars() { soc_pillar idh_pillar influxdb_pillar + postgres_pillar logrotate_pillar patch_pillar nginx_pillar @@ -998,6 +1049,11 @@ filter_unused_nics() { grep_string="$grep_string\|$BONDNIC" done fi + if [[ $MBNICS ]]; then + for BONDNIC in "${MBNICS[@]}"; do + grep_string="$grep_string\|$BONDNIC" + done + fi # Finally, set filtered_nics to any NICs we aren't using (and ignore interfaces that aren't of use) filtered_nics=$(ip link | awk -F: '$0 !~ "lo|vir|veth|br|docker|wl|^[^0-9]"{print $2}' | grep -vwe "$grep_string" | sed 's/ //g' | sed -r 's/(.*)(\.[0-9]+)@\1/\1\2/g') @@ -1053,6 +1109,7 @@ generate_passwords(){ HYDRAKEY=$(get_random_value) HYDRASALT=$(get_random_value) REDISPASS=$(get_random_value) + POSTGRESPASS=$(get_random_value) SOCSRVKEY=$(get_random_value 64) IMPORTPASS=$(get_random_value) } @@ -1355,6 +1412,12 @@ influxdb_pillar() { " token: $INFLUXTOKEN" > $local_salt_dir/pillar/influxdb/token.sls } +postgres_pillar() { + title "Create the postgres pillar file" + touch $adv_postgres_pillar_file + touch $postgres_pillar_file +} + make_some_dirs() { mkdir -p /nsm mkdir -p "$default_salt_dir" @@ -1364,7 +1427,7 @@ make_some_dirs() { mkdir -p $local_salt_dir/salt/firewall/portgroups mkdir -p $local_salt_dir/salt/firewall/ports - for THEDIR in bpf elasticsearch ntp firewall redis backup influxdb strelka sensoroni soc docker zeek suricata nginx telegraf logstash soc manager kratos hydra idh elastalert stig global kafka versionlock hypervisor vm; do + for THEDIR in bpf elasticsearch ntp firewall redis backup influxdb postgres strelka sensoroni soc docker zeek suricata nginx telegraf logstash soc manager kratos hydra idh elastalert stig global kafka versionlock hypervisor vm; do mkdir -p $local_salt_dir/pillar/$THEDIR touch $local_salt_dir/pillar/$THEDIR/adv_$THEDIR.sls touch $local_salt_dir/pillar/$THEDIR/soc_$THEDIR.sls @@ -1380,7 +1443,7 @@ network_init() { title "Initializing Network" disable_ipv6 set_hostname - if [[ ( $is_iso || $is_desktop_iso ) ]]; then + if [[ $is_iso || $is_desktop_iso ]]; then set_management_interface fi } @@ -1541,13 +1604,8 @@ clear_previous_setup_results() { reinstall_init() { info "Putting system in state to run setup again" - if [[ $install_type =~ ^(MANAGER|EVAL|MANAGERSEARCH|MANAGERHYPE|STANDALONE|FLEET|IMPORT)$ ]]; then - local salt_services=( "salt-master" "salt-minion" ) - else - local salt_services=( "salt-minion" ) - fi - - local service_retry_count=20 + # Always include both services. check_service_status skips units that aren't present. + local salt_services=( "salt-master" "salt-minion" ) { # remove all of root's cronjobs @@ -1563,31 +1621,51 @@ reinstall_init() { salt-call state.apply ca.remove -linfo --local --file-root=../salt - # Kill any salt processes (safely) + # Stop salt services and force-kill any lingering salt processes (including orphans + # from an earlier reinstall attempt where the unit file is gone but processes survive) + # so dnf remove salt can run cleanly for service in "${salt_services[@]}"; do - # Stop the service in the background so we can exit after a certain amount of time if check_service_status "$service"; then - systemctl stop "$service" & + info "Stopping $service via systemctl" + systemctl stop "$service" fi - local pid=$! - - local count=0 - while check_service_status "$service"; do - if [[ $count -gt $service_retry_count ]]; then - echo "Could not stop $service after 1 minute, exiting setup." - - # Stop the systemctl process trying to kill the service, show user a message, then exit setup - kill -9 $pid - fail_setup - fi - - sleep 5 - ((count++)) - done done + # Unconditionally force-kill any remaining salt binaries — these may be orphaned + # from a prior aborted reinstall (no unit file, so systemctl can't see them). + for salt_bin in salt-master salt-minion salt-call salt-cloud; do + if pgrep -f "/usr/bin/${salt_bin}" > /dev/null 2>&1; then + info "Force-killing lingering $salt_bin processes" + pkill -9 -ef "/usr/bin/${salt_bin}" 2>/dev/null + fi + done + # Catch stray `salt` CLI children from saltutil.kill_all_jobs / state.apply invocations + pkill -9 -ef "/usr/bin/python3 /bin/salt" 2>/dev/null + + # Give the kernel a moment to reap the killed processes before dnf removes the binaries + local kill_wait=0 + while pgrep -f "/usr/bin/salt-" > /dev/null 2>&1; do + if [[ $kill_wait -gt 10 ]]; then + info "Salt processes still present after SIGKILL + 10s wait; proceeding anyway" + pgrep -af "/usr/bin/salt-" | while read -r line; do info " lingering: $line"; done + break + fi + sleep 1 + ((kill_wait++)) + done + + # Clear the 'failed' state SIGKILL left on the units before removing the package + systemctl reset-failed salt-master.service salt-minion.service 2>/dev/null || true + # Remove all salt configs - rm -rf /etc/salt/engines/* /etc/salt/grains /etc/salt/master /etc/salt/master.d/* /etc/salt/minion /etc/salt/minion.d/* /etc/salt/pki/* /etc/salt/proxy /etc/salt/proxy.d/* /var/cache/salt/ + dnf -y remove salt + rm -rf /etc/salt/ /var/cache/salt/ + + # Drop systemd's in-memory references to the now-removed units + systemctl daemon-reload + + # Uninstall local Elastic Agent, if installed + elastic-agent uninstall -f if command -v docker &> /dev/null; then # Stop and remove all so-* containers so files can be changed with more safety @@ -1611,10 +1689,7 @@ reinstall_init() { backup_dir /nsm/hydra "$date_string" backup_dir /nsm/influxdb "$date_string" - # Uninstall local Elastic Agent, if installed - elastic-agent uninstall -f - - } >> "$setup_log" 2>&1 + } 2>&1 | tee -a "$setup_log" info "System reinstall init has been completed." } @@ -1681,6 +1756,24 @@ remove_package() { fi } +ensure_pyyaml() { + title "Ensuring python3-pyyaml is installed" + if rpm -q python3-pyyaml >/dev/null 2>&1; then + info "python3-pyyaml already installed" + return 0 + fi + info "python3-pyyaml not found, attempting to install" + set -o pipefail + dnf -y install python3-pyyaml 2>&1 | tee -a "$setup_log" + local result=$? + set +o pipefail + if [[ $result -ne 0 ]] || ! rpm -q python3-pyyaml >/dev/null 2>&1; then + error "Failed to install python3-pyyaml (exit=$result)" + fail_setup + fi + info "python3-pyyaml installed successfully" +} + # When updating the salt version, also update the version in securityonion-builds/images/iso-task/Dockerfile and salt/salt/master.defaults.yaml and salt/salt/minion.defaults.yaml # CAUTION! SALT VERSION UDDATES - READ BELOW # When updating the salt version, also update the version in: @@ -1832,7 +1925,8 @@ secrets_pillar(){ printf '%s\n'\ "secrets:"\ " import_pass: $IMPORTPASS"\ - " influx_pass: $INFLUXPASS" > $local_salt_dir/pillar/secrets.sls + " influx_pass: $INFLUXPASS"\ + " postgres_pass: $POSTGRESPASS" > $local_salt_dir/pillar/secrets.sls fi } @@ -2063,8 +2157,12 @@ set_initial_firewall_access() { # Set up the management interface on the ISO set_management_interface() { title "Setting up the main interface" + if [[ $MNIC == "bond1" ]]; then + configure_management_bond || fail_setup + fi + if [ "$address_type" = 'DHCP' ]; then - logCmd "nmcli con mod $MNIC connection.autoconnect yes" + logCmd "nmcli con mod $MNIC connection.autoconnect yes ipv4.method auto" logCmd "nmcli con up $MNIC" logCmd "nmcli -p connection show $MNIC" else diff --git a/setup/so-setup b/setup/so-setup index 823a379df..6c77e781c 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -66,6 +66,9 @@ set_timezone # Let's see what OS we are dealing with here detect_os +# Ensure python3-pyyaml is available before any code that may need so-yaml/PyYAML +ensure_pyyaml + # Check to see if this is the setup type of "desktop". is_desktop= @@ -219,6 +222,7 @@ if [ -n "$test_profile" ]; then WEBUSER=onionuser@somewhere.invalid WEBPASSWD1=0n10nus3r WEBPASSWD2=0n10nus3r + NODE_DESCRIPTION="${HOSTNAME} - ${install_type} - ${MSRVIP_OFFSET}" update_sudoers_for_testing fi diff --git a/setup/so-variables b/setup/so-variables index a0d7aadc1..975debf20 100644 --- a/setup/so-variables +++ b/setup/so-variables @@ -202,6 +202,12 @@ export influxdb_pillar_file adv_influxdb_pillar_file="$local_salt_dir/pillar/influxdb/adv_influxdb.sls" export adv_influxdb_pillar_file +postgres_pillar_file="$local_salt_dir/pillar/postgres/soc_postgres.sls" +export postgres_pillar_file + +adv_postgres_pillar_file="$local_salt_dir/pillar/postgres/adv_postgres.sls" +export adv_postgres_pillar_file + logrotate_pillar_file="$local_salt_dir/pillar/logrotate/soc_logrotate.sls" export logrotate_pillar_file diff --git a/setup/so-verify b/setup/so-verify index 8d23275ea..672ed70cc 100755 --- a/setup/so-verify +++ b/setup/so-verify @@ -71,7 +71,8 @@ log_has_errors() { grep -vE "remove_failed_vm.sls" | \ grep -vE "failed to copy: httpReadSeeker" | \ grep -vE "Error response from daemon: failed to resolve reference" | \ - grep -vE "log-.*-pipeline_failed_attempts" &> "$error_log" + grep -vE "log-.*-pipeline_failed_attempts" | \ + grep -vE " -v ON_ERROR_STOP=1" &> "$error_log" if [[ $? -eq 0 ]]; then # This function succeeds (returns 0) if errors are detected diff --git a/setup/so-whiptail b/setup/so-whiptail index 9a1d21150..6188d3d30 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -845,18 +845,99 @@ whiptail_management_nic() { [ -n "$TESTING" ] && return filter_unused_nics + local management_nic_options=( "${nic_list_management[@]}" ) + if [[ $is_iso || $is_desktop_iso ]]; then + management_nic_options+=( "BOND" "Configure a bonded management interface" ) + fi - MNIC=$(whiptail --title "$whiptail_title" --menu "Please select the NIC you would like to use for management.\n\nUse the arrow keys to move around and the Enter key to select." 20 75 12 "${nic_list_management[@]}" 3>&1 1>&2 2>&3 ) + MNIC=$(whiptail --title "$whiptail_title" --menu "Please select the NIC you would like to use for management.\n\nUse the arrow keys to move around and the Enter key to select." 20 75 12 "${management_nic_options[@]}" 3>&1 1>&2 2>&3 ) local exitstatus=$? whiptail_check_exitstatus $exitstatus while [ -z "$MNIC" ] do - MNIC=$(whiptail --title "$whiptail_title" --menu "Please select the NIC you would like to use for management.\n\nUse the arrow keys to move around and the Enter key to select." 22 75 12 "${nic_list_management[@]}" 3>&1 1>&2 2>&3 ) + MNIC=$(whiptail --title "$whiptail_title" --menu "Please select the NIC you would like to use for management.\n\nUse the arrow keys to move around and the Enter key to select." 22 75 12 "${management_nic_options[@]}" 3>&1 1>&2 2>&3 ) local exitstatus=$? whiptail_check_exitstatus $exitstatus done + if [[ $MNIC == "BOND" ]]; then + whiptail_management_bond + fi +} + +whiptail_management_bond() { + + [ -n "$TESTING" ] && return + + MBOND_MODE=$(whiptail --title "$whiptail_title" --menu \ + "Choose the bond mode for the management interface.\n\nThe management bond will be created as bond1." 20 75 7 \ + "active-backup" "One active NIC with failover (recommended)" \ + "balance-rr" "Round-robin transmit policy" \ + "balance-xor" "Transmit based on selected hash policy" \ + "broadcast" "Transmit everything on all slave interfaces" \ + "802.3ad" "Dynamic link aggregation (requires switch support)" \ + "balance-tlb" "Adaptive transmit load balancing" \ + "balance-alb" "Adaptive load balancing" 3>&1 1>&2 2>&3) + local exitstatus=$? + whiptail_check_exitstatus $exitstatus + + while [ -z "$MBOND_MODE" ] + do + MBOND_MODE=$(whiptail --title "$whiptail_title" --menu \ + "Choose the bond mode for the management interface.\n\nThe management bond will be created as bond1." 20 75 7 \ + "active-backup" "One active NIC with failover (recommended)" \ + "balance-rr" "Round-robin transmit policy" \ + "balance-xor" "Transmit based on selected hash policy" \ + "broadcast" "Transmit everything on all slave interfaces" \ + "802.3ad" "Dynamic link aggregation (requires switch support)" \ + "balance-tlb" "Adaptive transmit load balancing" \ + "balance-alb" "Adaptive load balancing" 3>&1 1>&2 2>&3) + local exitstatus=$? + whiptail_check_exitstatus $exitstatus + done + + whiptail_management_bond_nics + MNIC="bond1" + + export MBOND_MODE MNIC +} + +whiptail_management_bond_nics() { + + [ -n "$TESTING" ] && return + + MBNICS=() + filter_unused_nics + + MBNICS=$(whiptail --title "$whiptail_title" --checklist "Please add NICs to the Management Interface:" 20 75 12 "${nic_list[@]}" 3>&1 1>&2 2>&3) + local exitstatus=$? + whiptail_check_exitstatus $exitstatus + + while [ -z "$MBNICS" ] + do + MBNICS=$(whiptail --title "$whiptail_title" --checklist "Please add NICs to the Management Interface:" 20 75 12 "${nic_list[@]}" 3>&1 1>&2 2>&3) + local exitstatus=$? + whiptail_check_exitstatus $exitstatus + done + + MBNICS=$(echo "$MBNICS" | tr -d '"') + + IFS=' ' read -ra MBNICS <<< "$MBNICS" + + for bond_nic in "${MBNICS[@]}"; do + for dev_status in "${nmcli_dev_status_list[@]}"; do + if [[ $dev_status == "${bond_nic}:unmanaged" ]]; then + whiptail \ + --title "$whiptail_title" \ + --msgbox "$bond_nic is unmanaged by Network Manager. Please remove it from other network management tools then re-run setup." \ + 8 75 + exit + fi + done + done + + export MBNICS } whiptail_net_method() { diff --git a/sigs/securityonion-3.1.0-20260521.iso.sig b/sigs/securityonion-3.1.0-20260521.iso.sig new file mode 100644 index 000000000..af7564315 Binary files /dev/null and b/sigs/securityonion-3.1.0-20260521.iso.sig differ