Remove helpLink for influxdb in soc_global.yaml

Removed helpLink for influxdb from endgamehost configuration.

.github/.gitleaks.toml

@@ -1,546 +0,0 @@
-title = "gitleaks config"
-
-# Gitleaks rules are defined by regular expressions and entropy ranges.
-# Some secrets have unique signatures which make detecting those secrets easy.
-# Examples of those secrets would be GitLab Personal Access Tokens, AWS keys, and GitHub Access Tokens.
-# All these examples have defined prefixes like `glpat`, `AKIA`, `ghp_`, etc.
-#
-# Other secrets might just be a hash which means we need to write more complex rules to verify
-# that what we are matching is a secret.
-#
-# Here is an example of a semi-generic secret
-#
-#   discord_client_secret = "8dyfuiRyq=vVc3RRr_edRk-fK__JItpZ"
-#
-# We can write a regular expression to capture the variable name (identifier),
-# the assignment symbol (like '=' or ':='), and finally the actual secret.
-# The structure of a rule to match this example secret is below:
-#
-#                                                           Beginning string
-#                                                               quotation
-#                                                                   │            End string quotation
-#                                                                   │                      │
-#                                                                   ▼                      ▼
-#    (?i)(discord[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9=_\-]{32})['\"]
-#
-#                   ▲                              ▲                                ▲
-#                   │                              │                                │
-#                   │                              │                                │
-#              identifier                  assignment symbol
-#                                                                                Secret
-#
-[[rules]]
-id = "gitlab-pat"
-description = "GitLab Personal Access Token"
-regex = '''glpat-[0-9a-zA-Z\-\_]{20}'''
-
-[[rules]]
-id = "aws-access-token"
-description = "AWS"
-regex = '''(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}'''
-
-# Cryptographic keys
-[[rules]]
-id = "PKCS8-PK"
-description = "PKCS8 private key"
-regex = '''-----BEGIN PRIVATE KEY-----'''
-
-[[rules]]
-id = "RSA-PK"
-description = "RSA private key"
-regex = '''-----BEGIN RSA PRIVATE KEY-----'''
-
-[[rules]]
-id = "OPENSSH-PK"
-description = "SSH private key"
-regex = '''-----BEGIN OPENSSH PRIVATE KEY-----'''
-
-[[rules]]
-id = "PGP-PK"
-description = "PGP private key"
-regex = '''-----BEGIN PGP PRIVATE KEY BLOCK-----'''
-
-[[rules]]
-id = "github-pat"
-description = "GitHub Personal Access Token"
-regex = '''ghp_[0-9a-zA-Z]{36}'''
-
-[[rules]]
-id = "github-oauth"
-description = "GitHub OAuth Access Token"
-regex = '''gho_[0-9a-zA-Z]{36}'''
-
-[[rules]]
-id = "SSH-DSA-PK"
-description = "SSH (DSA) private key"
-regex = '''-----BEGIN DSA PRIVATE KEY-----'''
-
-[[rules]]
-id = "SSH-EC-PK"
-description = "SSH (EC) private key"
-regex = '''-----BEGIN EC PRIVATE KEY-----'''
-
-
-[[rules]]
-id = "github-app-token"
-description = "GitHub App Token"
-regex = '''(ghu|ghs)_[0-9a-zA-Z]{36}'''
-
-[[rules]]
-id = "github-refresh-token"
-description = "GitHub Refresh Token"
-regex = '''ghr_[0-9a-zA-Z]{76}'''
-
-[[rules]]
-id = "shopify-shared-secret"
-description = "Shopify shared secret"
-regex = '''shpss_[a-fA-F0-9]{32}'''
-
-[[rules]]
-id = "shopify-access-token"
-description = "Shopify access token"
-regex = '''shpat_[a-fA-F0-9]{32}'''
-
-[[rules]]
-id = "shopify-custom-access-token"
-description = "Shopify custom app access token"
-regex = '''shpca_[a-fA-F0-9]{32}'''
-
-[[rules]]
-id = "shopify-private-app-access-token"
-description = "Shopify private app access token"
-regex = '''shppa_[a-fA-F0-9]{32}'''
-
-[[rules]]
-id = "slack-access-token"
-description = "Slack token"
-regex = '''xox[baprs]-([0-9a-zA-Z]{10,48})?'''
-
-[[rules]]
-id = "stripe-access-token"
-description = "Stripe"
-regex = '''(?i)(sk|pk)_(test|live)_[0-9a-z]{10,32}'''
-
-[[rules]]
-id = "pypi-upload-token"
-description = "PyPI upload token"
-regex = '''pypi-AgEIcHlwaS5vcmc[A-Za-z0-9\-_]{50,1000}'''
-
-[[rules]]
-id = "gcp-service-account"
-description = "Google (GCP) Service-account"
-regex = '''\"type\": \"service_account\"'''
-
-[[rules]]
-id = "heroku-api-key"
-description = "Heroku API Key"
-regex = ''' (?i)(heroku[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([0-9A-F]{8}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{12})['\"]'''
-secretGroup = 3
-
-[[rules]]
-id = "slack-web-hook"
-description = "Slack Webhook"
-regex = '''https://hooks.slack.com/services/T[a-zA-Z0-9_]{8}/B[a-zA-Z0-9_]{8,12}/[a-zA-Z0-9_]{24}'''
-
-[[rules]]
-id = "twilio-api-key"
-description = "Twilio API Key"
-regex = '''SK[0-9a-fA-F]{32}'''
-
-[[rules]]
-id = "age-secret-key"
-description = "Age secret key"
-regex = '''AGE-SECRET-KEY-1[QPZRY9X8GF2TVDW0S3JN54KHCE6MUA7L]{58}'''
-
-[[rules]]
-id = "facebook-token"
-description = "Facebook token"
-regex = '''(?i)(facebook[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-f0-9]{32})['\"]'''
-secretGroup = 3
-
-[[rules]]
-id = "twitter-token"
-description = "Twitter token"
-regex = '''(?i)(twitter[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-f0-9]{35,44})['\"]'''
-secretGroup = 3
-
-[[rules]]
-id = "adobe-client-id"
-description = "Adobe Client ID (Oauth Web)"
-regex = '''(?i)(adobe[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-f0-9]{32})['\"]'''
-secretGroup = 3
-
-[[rules]]
-id = "adobe-client-secret"
-description = "Adobe Client Secret"
-regex = '''(p8e-)(?i)[a-z0-9]{32}'''
-
-[[rules]]
-id = "alibaba-access-key-id"
-description = "Alibaba AccessKey ID"
-regex = '''(LTAI)(?i)[a-z0-9]{20}'''
-
-[[rules]]
-id = "alibaba-secret-key"
-description = "Alibaba Secret Key"
-regex = '''(?i)(alibaba[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{30})['\"]'''
-secretGroup = 3
-
-[[rules]]
-id = "asana-client-id"
-description = "Asana Client ID"
-regex = '''(?i)(asana[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([0-9]{16})['\"]'''
-secretGroup = 3
-
-[[rules]]
-id = "asana-client-secret"
-description = "Asana Client Secret"
-regex = '''(?i)(asana[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{32})['\"]'''
-secretGroup = 3
-
-[[rules]]
-id = "atlassian-api-token"
-description = "Atlassian API token"
-regex = '''(?i)(atlassian[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{24})['\"]'''
-secretGroup = 3
-
-[[rules]]
-id = "bitbucket-client-id"
-description = "Bitbucket client ID"
-regex = '''(?i)(bitbucket[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{32})['\"]'''
-secretGroup = 3
-
-[[rules]]
-id = "bitbucket-client-secret"
-description = "Bitbucket client secret"
-regex = '''(?i)(bitbucket[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9_\-]{64})['\"]'''
-secretGroup = 3
-
-[[rules]]
-id = "beamer-api-token"
-description = "Beamer API token"
-regex = '''(?i)(beamer[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](b_[a-z0-9=_\-]{44})['\"]'''
-secretGroup = 3
-
-[[rules]]
-id = "clojars-api-token"
-description = "Clojars API token"
-regex = '''(CLOJARS_)(?i)[a-z0-9]{60}'''
-
-[[rules]]
-id = "contentful-delivery-api-token"
-description = "Contentful delivery API token"
-regex = '''(?i)(contentful[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9\-=_]{43})['\"]'''
-secretGroup = 3
-
-[[rules]]
-id = "databricks-api-token"
-description = "Databricks API token"
-regex = '''dapi[a-h0-9]{32}'''
-
-[[rules]]
-id = "discord-api-token"
-description = "Discord API key"
-regex = '''(?i)(discord[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-h0-9]{64})['\"]'''
-secretGroup = 3
-
-[[rules]]
-id = "discord-client-id"
-description = "Discord client ID"
-regex = '''(?i)(discord[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([0-9]{18})['\"]'''
-secretGroup = 3
-
-[[rules]]
-id = "discord-client-secret"
-description = "Discord client secret"
-regex = '''(?i)(discord[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9=_\-]{32})['\"]'''
-secretGroup = 3
-
-[[rules]]
-id = "doppler-api-token"
-description = "Doppler API token"
-regex = '''['\"](dp\.pt\.)(?i)[a-z0-9]{43}['\"]'''
-
-[[rules]]
-id = "dropbox-api-secret"
-description = "Dropbox API secret/key"
-regex = '''(?i)(dropbox[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{15})['\"]'''
-
-[[rules]]
-id = "dropbox--api-key"
-description = "Dropbox API secret/key"
-regex = '''(?i)(dropbox[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{15})['\"]'''
-
-[[rules]]
-id = "dropbox-short-lived-api-token"
-description = "Dropbox short lived API token"
-regex = '''(?i)(dropbox[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](sl\.[a-z0-9\-=_]{135})['\"]'''
-
-[[rules]]
-id = "dropbox-long-lived-api-token"
-description = "Dropbox long lived API token"
-regex = '''(?i)(dropbox[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"][a-z0-9]{11}(AAAAAAAAAA)[a-z0-9\-_=]{43}['\"]'''
-
-[[rules]]
-id = "duffel-api-token"
-description = "Duffel API token"
-regex = '''['\"]duffel_(test|live)_(?i)[a-z0-9_-]{43}['\"]'''
-
-[[rules]]
-id = "dynatrace-api-token"
-description = "Dynatrace API token"
-regex = '''['\"]dt0c01\.(?i)[a-z0-9]{24}\.[a-z0-9]{64}['\"]'''
-
-[[rules]]
-id = "easypost-api-token"
-description = "EasyPost API token"
-regex = '''['\"]EZAK(?i)[a-z0-9]{54}['\"]'''
-
-[[rules]]
-id = "easypost-test-api-token"
-description = "EasyPost test API token"
-regex = '''['\"]EZTK(?i)[a-z0-9]{54}['\"]'''
-
-[[rules]]
-id = "fastly-api-token"
-description = "Fastly API token"
-regex = '''(?i)(fastly[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9\-=_]{32})['\"]'''
-secretGroup = 3
-
-[[rules]]
-id = "finicity-client-secret"
-description = "Finicity client secret"
-regex = '''(?i)(finicity[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{20})['\"]'''
-secretGroup = 3
-
-[[rules]]
-id = "finicity-api-token"
-description = "Finicity API token"
-regex = '''(?i)(finicity[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-f0-9]{32})['\"]'''
-secretGroup = 3
-
-[[rules]]
-id = "flutterwave-public-key"
-description = "Flutterwave public key"
-regex = '''FLWPUBK_TEST-(?i)[a-h0-9]{32}-X'''
-
-[[rules]]
-id = "flutterwave-secret-key"
-description = "Flutterwave secret key"
-regex = '''FLWSECK_TEST-(?i)[a-h0-9]{32}-X'''
-
-[[rules]]
-id = "flutterwave-enc-key"
-description = "Flutterwave encrypted key"
-regex = '''FLWSECK_TEST[a-h0-9]{12}'''
-
-[[rules]]
-id = "frameio-api-token"
-description = "Frame.io API token"
-regex = '''fio-u-(?i)[a-z0-9\-_=]{64}'''
-
-[[rules]]
-id = "gocardless-api-token"
-description = "GoCardless API token"
-regex = '''['\"]live_(?i)[a-z0-9\-_=]{40}['\"]'''
-
-[[rules]]
-id = "grafana-api-token"
-description = "Grafana API token"
-regex = '''['\"]eyJrIjoi(?i)[a-z0-9\-_=]{72,92}['\"]'''
-
-[[rules]]
-id = "hashicorp-tf-api-token"
-description = "HashiCorp Terraform user/org API token"
-regex = '''['\"](?i)[a-z0-9]{14}\.atlasv1\.[a-z0-9\-_=]{60,70}['\"]'''
-
-[[rules]]
-id = "hubspot-api-token"
-description = "HubSpot API token"
-regex = '''(?i)(hubspot[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-h0-9]{8}-[a-h0-9]{4}-[a-h0-9]{4}-[a-h0-9]{4}-[a-h0-9]{12})['\"]'''
-secretGroup = 3
-
-[[rules]]
-id = "intercom-api-token"
-description = "Intercom API token"
-regex = '''(?i)(intercom[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9=_]{60})['\"]'''
-secretGroup = 3
-
-[[rules]]
-id = "intercom-client-secret"
-description = "Intercom client secret/ID"
-regex = '''(?i)(intercom[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-h0-9]{8}-[a-h0-9]{4}-[a-h0-9]{4}-[a-h0-9]{4}-[a-h0-9]{12})['\"]'''
-secretGroup = 3
-
-[[rules]]
-id = "ionic-api-token"
-description = "Ionic API token"
-regex = '''(?i)(ionic[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](ion_[a-z0-9]{42})['\"]'''
-
-[[rules]]
-id = "linear-api-token"
-description = "Linear API token"
-regex = '''lin_api_(?i)[a-z0-9]{40}'''
-
-[[rules]]
-id = "linear-client-secret"
-description = "Linear client secret/ID"
-regex = '''(?i)(linear[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-f0-9]{32})['\"]'''
-secretGroup = 3
-
-[[rules]]
-id = "lob-api-key"
-description = "Lob API Key"
-regex = '''(?i)(lob[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]((live|test)_[a-f0-9]{35})['\"]'''
-secretGroup = 3
-
-[[rules]]
-id = "lob-pub-api-key"
-description = "Lob Publishable API Key"
-regex = '''(?i)(lob[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]((test|live)_pub_[a-f0-9]{31})['\"]'''
-secretGroup = 3
-
-[[rules]]
-id = "mailchimp-api-key"
-description = "Mailchimp API key"
-regex = '''(?i)(mailchimp[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-f0-9]{32}-us20)['\"]'''
-secretGroup = 3
-
-[[rules]]
-id = "mailgun-private-api-token"
-description = "Mailgun private API token"
-regex = '''(?i)(mailgun[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](key-[a-f0-9]{32})['\"]'''
-secretGroup = 3
-
-[[rules]]
-id = "mailgun-pub-key"
-description = "Mailgun public validation key"
-regex = '''(?i)(mailgun[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](pubkey-[a-f0-9]{32})['\"]'''
-secretGroup = 3
-
-[[rules]]
-id = "mailgun-signing-key"
-description = "Mailgun webhook signing key"
-regex = '''(?i)(mailgun[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-h0-9]{32}-[a-h0-9]{8}-[a-h0-9]{8})['\"]'''
-secretGroup = 3
-
-[[rules]]
-id = "mapbox-api-token"
-description = "Mapbox API token"
-regex = '''(?i)(pk\.[a-z0-9]{60}\.[a-z0-9]{22})'''
-
-[[rules]]
-id = "messagebird-api-token"
-description = "MessageBird API token"
-regex = '''(?i)(messagebird[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{25})['\"]'''
-secretGroup = 3
-
-[[rules]]
-id = "messagebird-client-id"
-description = "MessageBird API client ID"
-regex = '''(?i)(messagebird[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-h0-9]{8}-[a-h0-9]{4}-[a-h0-9]{4}-[a-h0-9]{4}-[a-h0-9]{12})['\"]'''
-secretGroup = 3
-
-[[rules]]
-id = "new-relic-user-api-key"
-description = "New Relic user API Key"
-regex = '''['\"](NRAK-[A-Z0-9]{27})['\"]'''
-
-[[rules]]
-id = "new-relic-user-api-id"
-description = "New Relic user API ID"
-regex = '''(?i)(newrelic[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([A-Z0-9]{64})['\"]'''
-secretGroup = 3
-
-[[rules]]
-id = "new-relic-browser-api-token"
-description = "New Relic ingest browser API token"
-regex = '''['\"](NRJS-[a-f0-9]{19})['\"]'''
-
-[[rules]]
-id = "npm-access-token"
-description = "npm access token"
-regex = '''['\"](npm_(?i)[a-z0-9]{36})['\"]'''
-
-[[rules]]
-id = "planetscale-password"
-description = "PlanetScale password"
-regex = '''pscale_pw_(?i)[a-z0-9\-_\.]{43}'''
-
-[[rules]]
-id = "planetscale-api-token"
-description = "PlanetScale API token"
-regex = '''pscale_tkn_(?i)[a-z0-9\-_\.]{43}'''
-
-[[rules]]
-id = "postman-api-token"
-description = "Postman API token"
-regex = '''PMAK-(?i)[a-f0-9]{24}\-[a-f0-9]{34}'''
-
-[[rules]]
-id = "pulumi-api-token"
-description = "Pulumi API token"
-regex = '''pul-[a-f0-9]{40}'''
-
-[[rules]]
-id = "rubygems-api-token"
-description = "Rubygem API token"
-regex = '''rubygems_[a-f0-9]{48}'''
-
-[[rules]]
-id = "sendgrid-api-token"
-description = "SendGrid API token"
-regex = '''SG\.(?i)[a-z0-9_\-\.]{66}'''
-
-[[rules]]
-id = "sendinblue-api-token"
-description = "Sendinblue API token"
-regex = '''xkeysib-[a-f0-9]{64}\-(?i)[a-z0-9]{16}'''
-
-[[rules]]
-id = "shippo-api-token"
-description = "Shippo API token"
-regex = '''shippo_(live|test)_[a-f0-9]{40}'''
-
-[[rules]]
-id = "linkedin-client-secret"
-description = "LinkedIn Client secret"
-regex = '''(?i)(linkedin[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z]{16})['\"]'''
-secretGroup = 3
-
-[[rules]]
-id = "linkedin-client-id"
-description = "LinkedIn Client ID"
-regex = '''(?i)(linkedin[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{14})['\"]'''
-secretGroup = 3
-
-[[rules]]
-id = "twitch-api-token"
-description = "Twitch API token"
-regex = '''(?i)(twitch[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{30})['\"]'''
-secretGroup = 3
-
-[[rules]]
-id = "typeform-api-token"
-description = "Typeform API token"
-regex = '''(?i)(typeform[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}(tfp_[a-z0-9\-_\.=]{59})'''
-secretGroup = 3
-
-[[rules]]
-id = "generic-api-key"
-description = "Generic API Key"
-regex = '''(?i)((key|api[^Version]|token|secret|password)[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([0-9a-zA-Z\-_=]{8,64})['\"]'''
-entropy = 3.7
-secretGroup = 4
-
-
-[allowlist]
-description = "global allow lists"
-regexes = ['''219-09-9999''', '''078-05-1120''', '''(9[0-9]{2}|666)-\d{2}-\d{4}''', '''RPM-GPG-KEY.*''', '''.*:.*StrelkaHexDump.*''', '''.*:.*PLACEHOLDER.*''', '''ssl_.*password''', '''integration_key\s=\s"so-logs-"''']
-paths = [
-    '''gitleaks.toml''',
-    '''(.*?)(jpg|gif|doc|pdf|bin|svg|socket)$''',
-    '''(go.mod|go.sum)$''',
-    '''salt/nginx/files/enterprise-attack.json''',
-    '''(.*?)whl$'''
-]

.github/DISCUSSION_TEMPLATE/3-0.yml

@@ -10,6 +10,7 @@ body:
       options:
         -
         - 3.0.0
+        - 3.1.0
         - Other (please provide detail below)
     validations:
       required: true

.github/pull_request_template.md

@@ -0,0 +1,22 @@
+## Description
+
+<!-- 
+Explain the purpose of the pull request. Be brief or detailed depending on the scope of the changes.
+-->
+
+## Related Issues
+
+<!-- 
+Optionally, list any related issues that this pull request addresses.
+-->
+
+## Checklist
+
+- [ ] I have read and followed the [CONTRIBUTING.md](https://github.com/Security-Onion-Solutions/securityonion/blob/3/main/CONTRIBUTING.md) file.
+- [ ] I have read and agree to the terms of the [Contributor License Agreement](https://securityonionsolutions.com/cla)
+
+## Questions or Comments
+
+<!--
+If you have any questions or comments about this pull request, add them here.
+-->

.github/workflows/contrib.yml

@@ -1,24 +0,0 @@
-name: contrib
-on:
-  issue_comment:
-    types: [created]
-  pull_request_target:
-    types: [opened,closed,synchronize]
-
-jobs:
-  CLAssistant:
-    runs-on: ubuntu-latest
-    steps:
-      - name: "Contributor Check"
-        if: (github.event.comment.body == 'recheck' || github.event.comment.body == 'I have read the CLA Document and I hereby sign the CLA') || github.event_name == 'pull_request_target'
-        uses: cla-assistant/github-action@v2.3.1
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          PERSONAL_ACCESS_TOKEN : ${{ secrets.PERSONAL_ACCESS_TOKEN }}
-        with:
-          path-to-signatures: 'signatures_v1.json'
-          path-to-document: 'https://securityonionsolutions.com/cla' 
-          allowlist: dependabot[bot],jertel,dougburks,TOoSmOotH,defensivedepth,m0duspwnens
-          remote-organization-name: Security-Onion-Solutions
-          remote-repository-name: licensing
-

.github/workflows/leaktest.yml

@@ -1,17 +0,0 @@
-name: leak-test
-
-on: [pull_request]
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v2
-      with:
-        fetch-depth: '0'
-
-    - name: Gitleaks
-      uses: gitleaks/gitleaks-action@v1.6.0
-      with:
-        config-path: .github/.gitleaks.toml

CONTRIBUTING.md

@@ -23,7 +23,7 @@
 
 * Link the PR to the related issue, either using [keywords](https://docs.github.com/en/issues/tracking-your-work-with-issues/creating-issues/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword) in the PR description, or [manually](https://docs.github.com/en/issues/tracking-your-work-with-issues/creating-issues/linking-a-pull-request-to-an-issue#manually-linking-a-pull-request-to-an-issue).
 
-* **Pull requests should be opened against the `dev` branch of this repo**, and should clearly describe the problem and solution.
+* **Pull requests should be opened against the current `?/dev` branch of this repo**, and should clearly describe the problem and solution.
 
 * Be sure you have tested your changes and are confident they will not break other parts of the product.
 

DOWNLOAD_AND_VERIFY_ISO.md

@@ -1,46 +1,46 @@
-### 2.4.210-20260302 ISO image released on 2026/03/02
+### 3.0.0-20260331 ISO image released on 2026/03/31
 
 
 ### Download and Verify
 
-2.4.210-20260302 ISO image:  
-https://download.securityonion.net/file/securityonion/securityonion-2.4.210-20260302.iso
+3.0.0-20260331 ISO image:  
+https://download.securityonion.net/file/securityonion/securityonion-3.0.0-20260331.iso
  
-MD5: 575F316981891EBED2EE4E1F42A1F016  
-SHA1: 600945E8823221CBC5F1C056084A71355308227E  
-SHA256: A6AA6471125F07FA6E2796430E94BEAFDEF728E833E9728FDFA7106351EBC47E  
+MD5: ECD318A1662A6FDE0EF213F5A9BD4B07  
+SHA1: E55BE314440CCF3392DC0B06BC5E270B43176D9C  
+SHA256: 7FC47405E335CBE5C2B6C51FE7AC60248F35CBE504907B8B5A33822B23F8F4D5  
 
 Signature for ISO image:  
-https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.210-20260302.iso.sig
+https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.0.0-20260331.iso.sig
 
 Signing key:  
-https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.4/main/KEYS  
+https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/3/main/KEYS  
 
 For example, here are the steps you can use on most Linux distributions to download and verify our Security Onion ISO image.
 
 Download and import the signing key:  
 ```
-wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.4/main/KEYS -O - | gpg --import -  
+wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/3/main/KEYS -O - | gpg --import -  
 ```
 
 Download the signature file for the ISO:  
 ```
-wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.210-20260302.iso.sig
+wget https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.0.0-20260331.iso.sig
 ```
 
 Download the ISO image:  
 ```
-wget https://download.securityonion.net/file/securityonion/securityonion-2.4.210-20260302.iso
+wget https://download.securityonion.net/file/securityonion/securityonion-3.0.0-20260331.iso
 ```
 
 Verify the downloaded ISO image using the signature file:  
 ```
-gpg --verify securityonion-2.4.210-20260302.iso.sig securityonion-2.4.210-20260302.iso
+gpg --verify securityonion-3.0.0-20260331.iso.sig securityonion-3.0.0-20260331.iso
 ```
 
 The output should show "Good signature" and the Primary key fingerprint should match what's shown below:
 ```
-gpg: Signature made Mon 02 Mar 2026 11:55:24 AM EST using RSA key ID FE507013
+gpg: Signature made Mon 30 Mar 2026 06:22:14 PM EDT using RSA key ID FE507013
 gpg: Good signature from "Security Onion Solutions, LLC <info@securityonionsolutions.com>"
 gpg: WARNING: This key is not certified with a trusted signature!
 gpg:          There is no indication that the signature belongs to the owner.

VERSION

@@ -1 +1 @@
-3.0.0
+3.1.0

pillar/elasticsearch/index_templates.sls

@@ -1,2 +0,0 @@
-elasticsearch:
-  index_settings:

pillar/telegraf/creds.sls

@@ -0,0 +1,12 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+# Per-minion Telegraf Postgres credentials. so-telegraf-cred on the manager is
+# the single writer; it mutates /opt/so/saltstack/local/pillar/telegraf/creds.sls
+# under flock. Pillar_roots order (local before default) means the populated
+# copy shadows this default on any real grid; this file exists so the pillar
+# key is always defined on fresh installs and when no minions have creds yet.
+telegraf:
+  postgres_creds: {}

pillar/top.sls

@@ -17,6 +17,7 @@ base:
     - sensoroni.adv_sensoroni
     - telegraf.soc_telegraf
     - telegraf.adv_telegraf
+    - telegraf.creds
     - versionlock.soc_versionlock
     - versionlock.adv_versionlock
     - soc.license
@@ -38,6 +39,9 @@ base:
     {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
     - elasticsearch.auth
     {% endif %}
+    {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/postgres/auth.sls') %}
+    - postgres.auth
+    {% endif %}
     {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/kibana/secrets.sls') %}
     - kibana.secrets
     {% endif %}
@@ -60,6 +64,8 @@ base:
     - redis.adv_redis
     - influxdb.soc_influxdb
     - influxdb.adv_influxdb
+    - postgres.soc_postgres
+    - postgres.adv_postgres
     - elasticsearch.nodes
     - elasticsearch.soc_elasticsearch
     - elasticsearch.adv_elasticsearch
@@ -97,10 +103,12 @@ base:
     - node_data.ips
     - secrets
     - healthcheck.eval
-    - elasticsearch.index_templates
     {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
     - elasticsearch.auth
     {% endif %}
+    {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/postgres/auth.sls') %}
+    - postgres.auth
+    {% endif %}
     {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/kibana/secrets.sls') %}
     - kibana.secrets
     {% endif %}
@@ -126,6 +134,8 @@ base:
     - redis.adv_redis
     - influxdb.soc_influxdb
     - influxdb.adv_influxdb
+    - postgres.soc_postgres
+    - postgres.adv_postgres
     - backup.soc_backup
     - backup.adv_backup
     - zeek.soc_zeek
@@ -142,10 +152,12 @@ base:
     - logstash.nodes
     - logstash.soc_logstash
     - logstash.adv_logstash
-    - elasticsearch.index_templates
     {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
     - elasticsearch.auth
     {% endif %}
+    {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/postgres/auth.sls') %}
+    - postgres.auth
+    {% endif %}
     {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/kibana/secrets.sls') %}
     - kibana.secrets
     {% endif %}
@@ -160,6 +172,8 @@ base:
     - redis.adv_redis
     - influxdb.soc_influxdb
     - influxdb.adv_influxdb
+    - postgres.soc_postgres
+    - postgres.adv_postgres
     - elasticsearch.nodes
     - elasticsearch.soc_elasticsearch
     - elasticsearch.adv_elasticsearch
@@ -256,10 +270,12 @@ base:
   '*_import':
     - node_data.ips
     - secrets
-    - elasticsearch.index_templates
     {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
     - elasticsearch.auth
     {% endif %}
+    {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/postgres/auth.sls') %}
+    - postgres.auth
+    {% endif %}
     {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/kibana/secrets.sls') %}
     - kibana.secrets
     {% endif %}
@@ -285,6 +301,8 @@ base:
     - redis.adv_redis
     - influxdb.soc_influxdb
     - influxdb.adv_influxdb
+    - postgres.soc_postgres
+    - postgres.adv_postgres
     - zeek.soc_zeek
     - zeek.adv_zeek
     - bpf.soc_bpf

salt/allowed_states.map.jinja

@@ -29,6 +29,8 @@
     'manager',
     'nginx',
     'influxdb',
+    'postgres',
+    'postgres.auth',
     'soc',
     'kratos',
     'hydra',

salt/backup/config_backup.sls

@@ -32,3 +32,4 @@ so_config_backup:
     - daymonth: '*'
     - month: '*'
     - dayweek: '*'
+

salt/ca/files/signing_policies.conf

@@ -54,6 +54,20 @@ x509_signing_policies:
     - extendedKeyUsage: serverAuth
     - days_valid: 820
     - copypath: /etc/pki/issued_certs/
+  postgres:
+    - minions: '*'
+    - signing_private_key: /etc/pki/ca.key
+    - signing_cert: /etc/pki/ca.crt
+    - C: US
+    - ST: Utah
+    - L: Salt Lake City
+    - basicConstraints: "critical CA:false"
+    - keyUsage: "critical keyEncipherment"
+    - subjectKeyIdentifier: hash
+    - authorityKeyIdentifier: keyid,issuer:always
+    - extendedKeyUsage: serverAuth
+    - days_valid: 820
+    - copypath: /etc/pki/issued_certs/
   elasticfleet:
     - minions: '*'
     - signing_private_key: /etc/pki/ca.key

salt/common/tools/sbin/so-image-common

@@ -31,6 +31,7 @@ container_list() {
       "so-hydra"
       "so-nginx"
       "so-pcaptools"
+      "so-postgres"
       "so-soc"
       "so-suricata"
       "so-telegraf"
@@ -55,6 +56,7 @@ container_list() {
       "so-logstash"
       "so-nginx"
       "so-pcaptools"
+      "so-postgres"
       "so-redis"
       "so-soc"
       "so-strelka-backend"

salt/docker/defaults.yaml

@@ -237,3 +237,11 @@ docker:
       extra_hosts: []
       extra_env: []
       ulimits: []
+    'so-postgres':
+      final_octet: 47
+      port_bindings:
+        - 0.0.0.0:5432:5432
+      custom_bind_mounts: []
+      extra_hosts: []
+      extra_env: []
+      ulimits: []

salt/elasticfleet/content-defaults.map.jinja

@@ -0,0 +1,123 @@
+{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+  or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use
+  this file except in compliance with the Elastic License 2.0. #}
+
+
+{% import_json '/opt/so/state/esfleet_content_package_components.json' as ADDON_CONTENT_PACKAGE_COMPONENTS %}
+{% import_json '/opt/so/state/esfleet_component_templates.json' as INSTALLED_COMPONENT_TEMPLATES %}
+{% import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %}
+
+{% set CORE_ESFLEET_PACKAGES = ELASTICFLEETDEFAULTS.get('elasticfleet', {}).get('packages', {}) %}
+{% set ADDON_CONTENT_INTEGRATION_DEFAULTS = {} %}
+{% set DEBUG_STUFF = {} %}
+
+{% for pkg in ADDON_CONTENT_PACKAGE_COMPONENTS %}
+{%   if pkg.name in CORE_ESFLEET_PACKAGES %}
+{#     skip core content packages #}
+{%   elif pkg.name not in CORE_ESFLEET_PACKAGES %}
+{#     generate defaults for each content package #}
+{%     if pkg.dataStreams is defined and pkg.dataStreams is not none and pkg.dataStreams | length > 0%}
+{%       for pattern in pkg.dataStreams %}
+{#         in ES 9.3.2 'input' type integrations no longer create default component templates and instead they wait for user input during 'integration' setup (fleet ui config)
+             title: generic is an artifact of that and is not in use #}
+{%         if pattern.title == "generic" %}
+{%           continue %}
+{%         endif %}
+{%         if "metrics-" in pattern.name %}
+{%           set integration_type = "metrics-" %}
+{%         elif "logs-" in pattern.name %}
+{%           set integration_type = "logs-" %}
+{%         else %}
+{%           set integration_type = "" %}
+{%         endif %}
+{#         on content integrations the component name is user defined at the time it is added to an agent policy #}
+{%         set component_name = pattern.title %}
+{%         set index_pattern = pattern.name %}
+{#           component_name_x maintains the functionality of merging local pillar changes with generated 'defaults' via SOC UI #}
+{%           set component_name_x = component_name.replace(".","_x_") %}
+{#           pillar overrides/merge expects the key names to follow the naming in elasticsearch/defaults.yaml eg. so-logs-1password_x_item_usages . The _x_ is replaced later on in elasticsearch/template.map.jinja #}
+{%           set integration_key = "so-" ~ integration_type ~ pkg.name + '_x_' ~ component_name_x %}
+{#           Default integration settings #}
+{%           set integration_defaults = {
+               "index_sorting": false,
+               "index_template": {
+                 "composed_of": [integration_type ~ component_name ~ "@package", integration_type ~ component_name ~ "@custom", "so-fleet_integrations.ip_mappings-1", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"],
+                 "data_stream": {
+                   "allow_custom_routing": false,
+                   "hidden": false
+                 },
+                 "ignore_missing_component_templates": [integration_type ~ component_name ~ "@custom"],
+                 "index_patterns": [index_pattern],
+                 "priority": 501,
+                 "template": {
+                   "settings": {
+                     "index": {
+                       "lifecycle": {"name": "so-" ~ integration_type ~ component_name ~ "-logs"},
+                         "number_of_replicas": 0
+                     }
+                   }
+                 }
+               },
+               "policy": {
+                 "phases": {
+                   "cold": {
+                     "actions": {
+                        "allocate":{
+                          "number_of_replicas": ""
+                        },
+                       "set_priority": {"priority": 0}
+                     },
+                     "min_age": "60d"
+                   },
+                   "delete": {
+                     "actions": {
+                       "delete": {}
+                     },
+                     "min_age": "365d"
+                   },
+                   "hot": {
+                     "actions": {
+                       "rollover": {
+                         "max_age": "30d",
+                         "max_primary_shard_size": "50gb"
+                       },
+                       "forcemerge":{
+                         "max_num_segments": ""
+                       },
+                       "shrink":{
+                         "max_primary_shard_size": "",
+                         "method": "COUNT",
+                         "number_of_shards": ""
+                       },
+                       "set_priority": {"priority": 100}
+                      },
+                      "min_age": "0ms"
+                   },
+                   "warm": {
+                     "actions": {
+                        "allocate": {
+                          "number_of_replicas": ""
+                        },
+                        "forcemerge": {
+                          "max_num_segments": ""
+                        },
+                        "shrink":{
+                          "max_primary_shard_size": "",
+                          "method": "COUNT",
+                          "number_of_shards": ""
+                        },
+                       "set_priority": {"priority": 50}
+                     },
+                     "min_age": "30d"
+                   }
+                 }
+               }
+             } %}
+
+
+{%         do ADDON_CONTENT_INTEGRATION_DEFAULTS.update({integration_key: integration_defaults}) %}
+{%       endfor %}
+{%     else %}
+{%     endif %}
+{%   endif %}
+{% endfor %}

salt/elasticfleet/defaults.yaml

@@ -1,5 +1,6 @@
 elasticfleet:
   enabled: False
+  patch_version: 9.3.3+build202604082258 # Elastic Agent specific patch release.
   enable_manager_output: True
   config:
     server:

salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/import-zeek-logs.json

@@ -9,16 +9,22 @@
   "namespace": "so",
   "description": "Zeek Import logs",
   "policy_id": "so-grid-nodes_general",
+  "policy_ids": [
+    "so-grid-nodes_general"
+  ],
+  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.generic": {
+        "filestream.filestream": {
           "enabled": true,
           "vars": {
             "paths": [
               "/nsm/import/*/zeek/logs/*.log"
             ],
+            "compression_gzip": false,
+            "use_logs_stream": false,
             "data_stream.dataset": "import",
             "pipeline": "",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -34,7 +40,8 @@
             "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": []
+            "include_lines": [],
+            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/kratos-logs.json

@@ -15,19 +15,25 @@
     "version": ""
   },
   "name": "kratos-logs",
+  "namespace": "so",
   "description": "Kratos logs",
   "policy_id": "so-grid-nodes_general",
-  "namespace": "so",
+  "policy_ids": [
+    "so-grid-nodes_general"
+  ],
+  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.generic": {
+        "filestream.filestream": {
           "enabled": true,
           "vars": {
             "paths": [
               "/opt/so/log/kratos/kratos.log"
             ],
+            "compression_gzip": false,
+            "use_logs_stream": false,
             "data_stream.dataset": "kratos",
             "pipeline": "kratos",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -48,10 +54,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
-            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": []
+            "include_lines": [],
+            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/zeek-logs.json

@@ -9,16 +9,22 @@
   "namespace": "so",
   "description": "Zeek logs",
   "policy_id": "so-grid-nodes_general",
+  "policy_ids": [
+    "so-grid-nodes_general"
+  ],
+  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.generic": {
+        "filestream.filestream": {
           "enabled": true,
           "vars": {
             "paths": [
               "/nsm/zeek/logs/current/*.log"
             ],
+            "compression_gzip": false,
+            "use_logs_stream": false,
             "data_stream.dataset": "zeek",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
             "exclude_files": ["({%- endraw -%}{{ ELASTICFLEETMERGED.logging.zeek.excluded | join('|') }}{%- raw -%})(\\..+)?\\.log$"],
@@ -30,10 +36,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
-            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": []
+            "include_lines": [],
+            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json

@@ -5,7 +5,7 @@
   "package": {
     "name": "endpoint",
     "title": "Elastic Defend",
-    "version": "9.0.2",
+    "version": "9.3.0",
     "requires_root": true
   },
   "enabled": true,

salt/elasticfleet/files/integrations/grid-nodes_general/elastic-agent-monitor.json

@@ -6,21 +6,23 @@
   "name": "agent-monitor",
   "namespace": "",
   "description": "",
+  "policy_id": "so-grid-nodes_general",
   "policy_ids": [
     "so-grid-nodes_general"
   ],
-  "output_id": null,
   "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.generic": {
+        "filestream.filestream": {
           "enabled": true,
           "vars": {
             "paths": [
               "/opt/so/log/agents/agent-monitor.log"
             ],
+            "compression_gzip": false,
+            "use_logs_stream": false,
             "data_stream.dataset": "agentmonitor",
             "pipeline": "elasticagent.monitor",
             "parsers": "",
@@ -34,15 +36,16 @@
             "ignore_older": "72h",
             "clean_inactive": -1,
             "harvester_limit": 0,
-            "fingerprint": true,
+            "fingerprint": false,
             "fingerprint_offset": 0,
-            "fingerprint_length": 64,
-            "file_identity_native": false,
+            "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": []
-          }
+            "include_lines": [],
+            "delete_enabled": false
           }
         }
       }
     }
+  },
+  "force": true
 }

salt/elasticfleet/files/integrations/grid-nodes_general/hydra-logs.json

@@ -4,19 +4,25 @@
     "version": ""
   },
   "name": "hydra-logs",
+  "namespace": "so",
   "description": "Hydra logs",
   "policy_id": "so-grid-nodes_general",
-  "namespace": "so",
+  "policy_ids": [
+    "so-grid-nodes_general"
+  ],
+  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.generic": {
+        "filestream.filestream": {
           "enabled": true,
           "vars": {
             "paths": [
               "/opt/so/log/hydra/hydra.log"
             ],
+            "compression_gzip": false,
+            "use_logs_stream": false,
             "data_stream.dataset": "hydra",
             "pipeline": "hydra",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -34,10 +40,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
-            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": []
+            "include_lines": [],
+            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/idh-logs.json

@@ -4,19 +4,25 @@
     "version": ""
   },
   "name": "idh-logs",
+  "namespace": "so",
   "description": "IDH integration",
   "policy_id": "so-grid-nodes_general",
-  "namespace": "so",
+  "policy_ids": [
+    "so-grid-nodes_general"
+  ],
+  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.generic": {
+        "filestream.filestream": {
           "enabled": true,
           "vars": {
             "paths": [
               "/nsm/idh/opencanary.log"
             ],
+            "compression_gzip": false,
+            "use_logs_stream": false,
             "data_stream.dataset": "idh",
             "pipeline": "common",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -31,10 +37,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
-            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": []
+            "include_lines": [],
+            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json

@@ -4,26 +4,32 @@
     "version": ""
   },
   "name": "import-evtx-logs",
+  "namespace": "so",
   "description": "Import Windows EVTX logs",
   "policy_id": "so-grid-nodes_general",
-  "namespace": "so",
+  "policy_ids": [
+    "so-grid-nodes_general"
+  ],
+  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.generic": {
+        "filestream.filestream": {
           "enabled": true,
           "vars": {
             "paths": [
               "/nsm/import/*/evtx/*.json"
             ],
+            "compression_gzip": false,
+            "use_logs_stream": false,
             "data_stream.dataset": "import",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
             "exclude_files": [
               "\\.gz$"
             ],
             "include_files": [],
-            "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- drop_fields:\n    fields: [\"host\"]\n    ignore_missing: true\n- add_fields:\n    target: data_stream\n    fields:\n      type: logs\n      dataset: system.security\n- add_fields:\n    target: event\n    fields:\n      dataset: system.security\n      module: system\n      imported: true\n- add_fields:\n    target: \"@metadata\"\n    fields:\n        pipeline: logs-system.security-2.6.1\n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.sysmon_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.sysmon_operational\n          module: windows\n          imported: true\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.sysmon_operational-3.1.2\n- if:\n    equals:\n      winlog.channel: 'Application'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.application-2.6.1\n- if:\n    equals:\n      winlog.channel: 'System'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.system-2.6.1\n    \n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.powershell_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell_operational\n          module: windows\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.powershell_operational-3.1.2\n- add_fields:\n    target: data_stream\n    fields:\n        dataset: import",
+            "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- drop_fields:\n    fields: [\"host\"]\n    ignore_missing: true\n- add_fields:\n    target: data_stream\n    fields:\n      type: logs\n      dataset: system.security\n- add_fields:\n    target: event\n    fields:\n      dataset: system.security\n      module: system\n      imported: true\n- add_fields:\n    target: \"@metadata\"\n    fields:\n        pipeline: logs-system.security-2.15.0\n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.sysmon_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.sysmon_operational\n          module: windows\n          imported: true\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.sysmon_operational-3.8.0\n- if:\n    equals:\n      winlog.channel: 'Application'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.application-2.15.0\n- if:\n    equals:\n      winlog.channel: 'System'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.system-2.15.0\n    \n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.powershell_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell_operational\n          module: windows\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.powershell_operational-3.8.0\n- add_fields:\n    target: data_stream\n    fields:\n        dataset: import",
             "tags": [
               "import"
             ],
@@ -33,10 +39,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
-            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": []
+            "include_lines": [],
+            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/import-suricata-logs.json

@@ -4,19 +4,25 @@
     "version": ""
   },
   "name": "import-suricata-logs",
+  "namespace": "so",
   "description": "Import Suricata logs",
   "policy_id": "so-grid-nodes_general",
-  "namespace": "so",
+  "policy_ids": [
+    "so-grid-nodes_general"
+  ],
+  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.generic": {
+        "filestream.filestream": {
           "enabled": true,
           "vars": {
             "paths": [
               "/nsm/import/*/suricata/eve*.json"
             ],
+            "compression_gzip": false,
+            "use_logs_stream": false,
             "data_stream.dataset": "import",
             "pipeline": "suricata.common",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -32,10 +38,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
-            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": []
+            "include_lines": [],
+            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/rita-logs.json

@@ -4,14 +4,18 @@
     "version": ""
   },
   "name": "rita-logs",
+  "namespace": "so",
   "description": "RITA Logs",
   "policy_id": "so-grid-nodes_general",
-  "namespace": "so",
+  "policy_ids": [
+    "so-grid-nodes_general"
+  ],
+  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.generic": {
+        "filestream.filestream": {
           "enabled": true,
           "vars": {
             "paths": [
@@ -19,6 +23,8 @@
               "/nsm/rita/exploded-dns.csv",
               "/nsm/rita/long-connections.csv"
             ],
+            "compression_gzip": false,
+            "use_logs_stream": false,
             "data_stream.dataset": "rita",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
             "exclude_files": [
@@ -33,10 +39,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
-            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": []
+            "include_lines": [],
+            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/so-ip-mappings.json

@@ -4,19 +4,25 @@
     "version": ""
   },
   "name": "so-ip-mappings",
+  "namespace": "so",
   "description": "IP Description mappings",
   "policy_id": "so-grid-nodes_general",
-  "namespace": "so",
+  "policy_ids": [
+    "so-grid-nodes_general"
+  ],
+  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.generic": {
+        "filestream.filestream": {
           "enabled": true,
           "vars": {
             "paths": [
               "/nsm/custom-mappings/ip-descriptions.csv"
             ],
+            "compression_gzip": false,
+            "use_logs_stream": false,
             "data_stream.dataset": "hostnamemappings",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
             "exclude_files": [
@@ -32,10 +38,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
-            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": []
+            "include_lines": [],
+            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/soc-auth-sync-logs.json

@@ -4,19 +4,25 @@
     "version": ""
   },
   "name": "soc-auth-sync-logs",
+  "namespace": "so",
   "description": "Security Onion - Elastic Auth Sync - Logs",
   "policy_id": "so-grid-nodes_general",
-  "namespace": "so",
+  "policy_ids": [
+    "so-grid-nodes_general"
+  ],
+  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.generic": {
+        "filestream.filestream": {
           "enabled": true,
           "vars": {
             "paths": [
               "/opt/so/log/soc/sync.log"
             ],
+            "compression_gzip": false,
+            "use_logs_stream": false,
             "data_stream.dataset": "soc",
             "pipeline": "common",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -31,10 +37,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
-            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": []
+            "include_lines": [],
+            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/soc-detections-logs.json

@@ -4,20 +4,26 @@
     "version": ""
   },
   "name": "soc-detections-logs",
+  "namespace": "so",
   "description": "Security Onion Console - Detections Logs",
   "policy_id": "so-grid-nodes_general",
-  "namespace": "so",
+  "policy_ids": [
+    "so-grid-nodes_general"
+  ],
+  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.generic": {
+        "filestream.filestream": {
           "enabled": true,
           "vars": {
             "paths": [
               "/opt/so/log/soc/detections_runtime-status_sigma.log",
               "/opt/so/log/soc/detections_runtime-status_yara.log"
             ],
+            "compression_gzip": false,
+            "use_logs_stream": false,
             "data_stream.dataset": "soc",
             "pipeline": "common",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -35,10 +41,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
-            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": []
+            "include_lines": [],
+            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/soc-salt-relay-logs.json

@@ -4,19 +4,25 @@
     "version": ""
   },
   "name": "soc-salt-relay-logs",
+  "namespace": "so",
   "description": "Security Onion - Salt Relay - Logs",
   "policy_id": "so-grid-nodes_general",
-  "namespace": "so",
+  "policy_ids": [
+    "so-grid-nodes_general"
+  ],
+  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.generic": {
+        "filestream.filestream": {
           "enabled": true,
           "vars": {
             "paths": [
               "/opt/so/log/soc/salt-relay.log"
             ],
+            "compression_gzip": false,
+            "use_logs_stream": false,
             "data_stream.dataset": "soc",
             "pipeline": "common",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -33,10 +39,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
-            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": []
+            "include_lines": [],
+            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/soc-sensoroni-logs.json

@@ -4,19 +4,25 @@
     "version": ""
   },
   "name": "soc-sensoroni-logs",
+  "namespace": "so",
   "description": "Security Onion - Sensoroni - Logs",
   "policy_id": "so-grid-nodes_general",
-  "namespace": "so",
+  "policy_ids": [
+    "so-grid-nodes_general"
+  ],
+  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.generic": {
+        "filestream.filestream": {
           "enabled": true,
           "vars": {
             "paths": [
               "/opt/so/log/sensoroni/sensoroni.log"
             ],
+            "compression_gzip": false,
+            "use_logs_stream": false,
             "data_stream.dataset": "soc",
             "pipeline": "common",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -31,10 +37,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
-            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": []
+            "include_lines": [],
+            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/soc-server-logs.json

@@ -4,19 +4,25 @@
     "version": ""
   },
   "name": "soc-server-logs",
+  "namespace": "so",
   "description": "Security Onion Console Logs",
   "policy_id": "so-grid-nodes_general",
-  "namespace": "so",
+  "policy_ids": [
+    "so-grid-nodes_general"
+  ],
+  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.generic": {
+        "filestream.filestream": {
           "enabled": true,
           "vars": {
             "paths": [
               "/opt/so/log/soc/sensoroni-server.log"
             ],
+            "compression_gzip": false,
+            "use_logs_stream": false,
             "data_stream.dataset": "soc",
             "pipeline": "common",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -33,10 +39,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
-            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": []
+            "include_lines": [],
+            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/strelka-logs.json

@@ -4,19 +4,25 @@
     "version": ""
   },
   "name": "strelka-logs",
+  "namespace": "so",
   "description": "Strelka Logs",
   "policy_id": "so-grid-nodes_general",
-  "namespace": "so",
+  "policy_ids": [
+    "so-grid-nodes_general"
+  ],
+  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.generic": {
+        "filestream.filestream": {
           "enabled": true,
           "vars": {
             "paths": [
               "/nsm/strelka/log/strelka.log"
             ],
+            "compression_gzip": false,
+            "use_logs_stream": false,
             "data_stream.dataset": "strelka",
             "pipeline": "strelka.file",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -31,10 +37,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
-            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": []
+            "include_lines": [],
+            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/suricata-logs.json

@@ -4,19 +4,25 @@
     "version": ""
   },
   "name": "suricata-logs",
+  "namespace": "so",
   "description": "Suricata integration",
   "policy_id": "so-grid-nodes_general",
-  "namespace": "so",
+  "policy_ids": [
+    "so-grid-nodes_general"
+  ],
+  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.generic": {
+        "filestream.filestream": {
           "enabled": true,
           "vars": {
             "paths": [
               "/nsm/suricata/eve*.json"
             ],
+            "compression_gzip": false,
+            "use_logs_stream": false,
             "data_stream.dataset": "suricata",
             "pipeline": "suricata.common",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -31,10 +37,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
-            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": []
+            "include_lines": [],
+            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/input-defaults.map.jinja

@@ -0,0 +1,123 @@
+{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+  or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use
+  this file except in compliance with the Elastic License 2.0. #}
+
+
+{% import_json '/opt/so/state/esfleet_input_package_components.json' as ADDON_INPUT_PACKAGE_COMPONENTS %}
+{% import_json '/opt/so/state/esfleet_component_templates.json' as INSTALLED_COMPONENT_TEMPLATES %}
+{% import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %}
+
+{% set CORE_ESFLEET_PACKAGES = ELASTICFLEETDEFAULTS.get('elasticfleet', {}).get('packages', {}) %}
+{% set ADDON_INPUT_INTEGRATION_DEFAULTS = {} %}
+{% set DEBUG_STUFF = {} %}
+
+{% for pkg in ADDON_INPUT_PACKAGE_COMPONENTS %}
+{%   if pkg.name in CORE_ESFLEET_PACKAGES %}
+{#     skip core input packages #}
+{%   elif pkg.name not in CORE_ESFLEET_PACKAGES %}
+{#     generate defaults for each input package #}
+{%     if pkg.dataStreams is defined and pkg.dataStreams is not none and pkg.dataStreams | length > 0 %}
+{%       for pattern in pkg.dataStreams %}
+{#         in ES 9.3.2 'input' type integrations no longer create default component templates and instead they wait for user input during 'integration' setup (fleet ui config)
+             title: generic is an artifact of that and is not in use #}
+{%         if pattern.title == "generic" %}
+{%           continue %}
+{%         endif %}
+{%         if "metrics-" in pattern.name %}
+{%           set integration_type = "metrics-" %}
+{%         elif "logs-" in pattern.name %}
+{%           set integration_type = "logs-" %}
+{%         else %}
+{%           set integration_type = "" %}
+{%         endif %}
+{#         on input integrations the component name is user defined at the time it is added to an agent policy #}
+{%         set component_name = pattern.title %}
+{%         set index_pattern = pattern.name %}
+{#           component_name_x maintains the functionality of merging local pillar changes with generated 'defaults' via SOC UI #}
+{%           set component_name_x = component_name.replace(".","_x_") %}
+{#           pillar overrides/merge expects the key names to follow the naming in elasticsearch/defaults.yaml eg. so-logs-1password_x_item_usages . The _x_ is replaced later on in elasticsearch/template.map.jinja #}
+{%           set integration_key = "so-" ~ integration_type ~ pkg.name + '_x_' ~ component_name_x %}
+{#           Default integration settings #}
+{%           set integration_defaults = {
+               "index_sorting": false,
+               "index_template": {
+                 "composed_of": [integration_type ~ component_name ~ "@package", integration_type ~ component_name ~ "@custom", "so-fleet_integrations.ip_mappings-1", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"],
+                 "data_stream": {
+                   "allow_custom_routing": false,
+                   "hidden": false
+                 },
+                 "ignore_missing_component_templates": [integration_type ~ component_name ~ "@custom"],
+                 "index_patterns": [index_pattern],
+                 "priority": 501,
+                 "template": {
+                   "settings": {
+                     "index": {
+                       "lifecycle": {"name": "so-" ~ integration_type ~ component_name ~ "-logs"},
+                         "number_of_replicas": 0
+                     }
+                   }
+                 }
+               },
+               "policy": {
+                 "phases": {
+                   "cold": {
+                     "actions": {
+                        "allocate":{
+                          "number_of_replicas": ""
+                        },
+                       "set_priority": {"priority": 0}
+                     },
+                     "min_age": "60d"
+                   },
+                   "delete": {
+                     "actions": {
+                       "delete": {}
+                     },
+                     "min_age": "365d"
+                   },
+                   "hot": {
+                     "actions": {
+                       "rollover": {
+                         "max_age": "30d",
+                         "max_primary_shard_size": "50gb"
+                       },
+                       "forcemerge":{
+                         "max_num_segments": ""
+                       },
+                       "shrink":{
+                         "max_primary_shard_size": "",
+                         "method": "COUNT",
+                         "number_of_shards": ""
+                       },
+                       "set_priority": {"priority": 100}
+                      },
+                      "min_age": "0ms"
+                   },
+                   "warm": {
+                     "actions": {
+                        "allocate": {
+                          "number_of_replicas": ""
+                        },
+                        "forcemerge": {
+                          "max_num_segments": ""
+                        },
+                        "shrink":{
+                          "max_primary_shard_size": "",
+                          "method": "COUNT",
+                          "number_of_shards": ""
+                        },
+                       "set_priority": {"priority": 50}
+                     },
+                     "min_age": "30d"
+                   }
+                 }
+               }
+             } %}
+
+
+{%         do ADDON_INPUT_INTEGRATION_DEFAULTS.update({integration_key: integration_defaults}) %}
+{%         do DEBUG_STUFF.update({integration_key: "Generating defaults for "+ pkg.name })%}
+{%       endfor %}
+{%     endif %}
+{%   endif %}
+{% endfor %}

salt/elasticfleet/integration-defaults.map.jinja

@@ -59,8 +59,8 @@
 {#     skip core integrations #}
 {%   elif pkg.name not in CORE_ESFLEET_PACKAGES %}
 {#     generate defaults for each integration #}
-{%     if pkg.es_index_patterns is defined and pkg.es_index_patterns is not none %}
-{%       for pattern in pkg.es_index_patterns %}
+{%     if pkg.dataStreams is defined and pkg.dataStreams is not none and pkg.dataStreams | length > 0 %}
+{%       for pattern in pkg.dataStreams %}
 {%         if "metrics-" in pattern.name %}
 {%           set integration_type = "metrics-" %}
 {%         elif "logs-" in pattern.name %}
@@ -75,44 +75,27 @@
 {%         if component_name in WEIRD_INTEGRATIONS %}
 {%           set component_name = WEIRD_INTEGRATIONS[component_name] %}
 {%         endif %}
-
-{#         create duplicate of component_name, so we can split generics from @custom component templates in the index template below and overwrite the default @package when needed
-           eg. having to replace unifiedlogs.generic@package with filestream.generic@package, but keep the ability to customize unifiedlogs.generic@custom and its ILM policy #}
-{%         set custom_component_name = component_name %}
-
-{#         duplicate integration_type to assist with sometimes needing to overwrite component templates with 'logs-filestream.generic@package' (there is no metrics-filestream.generic@package) #}
-{%         set generic_integration_type = integration_type %}
-
 {#           component_name_x maintains the functionality of merging local pillar changes with generated 'defaults' via SOC UI #}
 {%           set component_name_x = component_name.replace(".","_x_") %}
 {#           pillar overrides/merge expects the key names to follow the naming in elasticsearch/defaults.yaml eg. so-logs-1password_x_item_usages . The _x_ is replaced later on in elasticsearch/template.map.jinja #}
 {%           set integration_key = "so-" ~ integration_type ~ component_name_x %}
 
-{#         if its a .generic template make sure that a .generic@package for the integration exists. Else default to logs-filestream.generic@package #}
-{%         if ".generic" in component_name and integration_type ~ component_name ~ "@package" not in INSTALLED_COMPONENT_TEMPLATES %}
-{#           these generic templates by default are directed to index_pattern of 'logs-generic-*', overwrite that here to point to eg gcp_pubsub.generic-* #}
-{%           set index_pattern = integration_type ~ component_name ~ "-*" %}
-{#           includes use of .generic component template, but it doesn't exist in installed component templates. Redirect it to filestream.generic@package #}
-{%           set component_name = "filestream.generic" %}
-{%           set generic_integration_type = "logs-" %}
-{%         endif %}
-
 {#           Default integration settings #}
 {%           set integration_defaults = {
                "index_sorting": false,
                "index_template": {
-                 "composed_of": [generic_integration_type ~ component_name ~ "@package", integration_type ~ custom_component_name ~ "@custom", "so-fleet_integrations.ip_mappings-1", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"],
+                 "composed_of": [integration_type ~ component_name ~ "@package", integration_type ~ component_name ~ "@custom", "so-fleet_integrations.ip_mappings-1", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"],
                  "data_stream": {
                    "allow_custom_routing": false,
                    "hidden": false
                  },
-                 "ignore_missing_component_templates": [integration_type ~ custom_component_name ~ "@custom"],
+                 "ignore_missing_component_templates": [integration_type ~ component_name ~ "@custom"],
                  "index_patterns": [index_pattern],
                  "priority": 501,
                  "template": {
                    "settings": {
                      "index": {
-                       "lifecycle": {"name": "so-" ~ integration_type ~ custom_component_name ~ "-logs"},
+                       "lifecycle": {"name": "so-" ~ integration_type ~ component_name ~ "-logs"},
                          "number_of_replicas": 0
                      }
                    }

salt/elasticfleet/tools/sbin/so-elastic-fleet-common

@@ -135,9 +135,33 @@ elastic_fleet_bulk_package_install() {
     fi
 }
 
-elastic_fleet_installed_packages() {
-    if ! fleet_api "epm/packages/installed?perPage=500"; then
+elastic_fleet_get_package_list_by_type() {
+    if ! output=$(fleet_api "epm/packages"); then
         return 1
+    else
+        is_integration=$(jq '[.items[] | select(.type=="integration") | .name ]' <<< "$output")
+        is_input=$(jq '[.items[] | select(.type=="input") | .name ]' <<< "$output")
+        is_content=$(jq '[.items[] | select(.type=="content") | .name ]' <<< "$output")
+        jq -n --argjson is_integration "${is_integration:-[]}" \
+              --argjson is_input "${is_input:-[]}" \
+              --argjson is_content "${is_content:-[]}" \
+              '{"integration": $is_integration,"input": $is_input, "content": $is_content}'
+    fi
+}
+elastic_fleet_installed_packages_components() {
+    package_type=${1,,}
+    if [[ "$package_type" != "integration" && "$package_type" != "input" && "$package_type" != "content" ]]; then
+        echo "Error: Invalid package type ${package_type}. Valid types are 'integration', 'input', or 'content'."
+        return 1
+    fi
+
+    packages_by_type=$(elastic_fleet_get_package_list_by_type)
+    packages=$(jq --arg package_type "$package_type" '.[$package_type]' <<< "$packages_by_type")
+
+    if ! output=$(fleet_api "epm/packages/installed?perPage=500"); then
+        return 1
+    else
+        jq -c --argjson packages "$packages" '[.items[] | select(.name | IN($packages[])) | {name: .name, dataStreams: .dataStreams}]' <<< "$output"
     fi
 }
 

salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-grid-upgrade

@@ -6,6 +6,11 @@
 
 . /usr/sbin/so-common
 {%- import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS %}
+{%- import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %}
+{# Optionally override Elasticsearch version for Elastic Agent patch releases #}
+{%- if ELASTICFLEETDEFAULTS.elasticfleet.patch_version is defined %}
+{%-   do ELASTICSEARCHDEFAULTS.update({'elasticsearch': {'version': ELASTICFLEETDEFAULTS.elasticfleet.patch_version}}) %}
+{%- endif %}
 
 # Only run on Managers
 if ! is_manager_node; then

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load

@@ -18,7 +18,9 @@ INSTALLED_PACKAGE_LIST=/tmp/esfleet_installed_packages.json
 BULK_INSTALL_PACKAGE_LIST=/tmp/esfleet_bulk_install.json
 BULK_INSTALL_PACKAGE_TMP=/tmp/esfleet_bulk_install_tmp.json
 BULK_INSTALL_OUTPUT=/opt/so/state/esfleet_bulk_install_results.json
-PACKAGE_COMPONENTS=/opt/so/state/esfleet_package_components.json
+INTEGRATION_PACKAGE_COMPONENTS=/opt/so/state/esfleet_package_components.json
+INPUT_PACKAGE_COMPONENTS=/opt/so/state/esfleet_input_package_components.json
+CONTENT_PACKAGE_COMPONENTS=/opt/so/state/esfleet_content_package_components.json
 COMPONENT_TEMPLATES=/opt/so/state/esfleet_component_templates.json
 
 PENDING_UPDATE=false
@@ -179,10 +181,13 @@ if [[ -f $STATE_FILE_SUCCESS  ]]; then
         else
             echo "Elastic integrations don't appear to need installation/updating..."
         fi
-        # Write out file for generating index/component/ilm templates
-        if latest_installed_package_list=$(elastic_fleet_installed_packages); then
-            echo $latest_installed_package_list | jq '[.items[] | {name: .name, es_index_patterns: .dataStreams}]' > $PACKAGE_COMPONENTS
+        # Write out file for generating index/component/ilm templates, keeping each package type separate
+        for package_type in "INTEGRATION" "INPUT" "CONTENT"; do
+            if latest_installed_package_list=$(elastic_fleet_installed_packages_components "$package_type"); then
+                outfile="${package_type}_PACKAGE_COMPONENTS"
+                echo $latest_installed_package_list > "${!outfile}"
             fi
+        done
         if retry 3 1 "so-elasticsearch-query / --fail --output /dev/null"; then
             # Refresh installed component template list
             latest_component_templates_list=$(so-elasticsearch-query _component_template | jq '.component_templates[] | .name' | jq -s '.')

salt/elasticsearch/cluster.sls

@@ -0,0 +1,164 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{% from 'allowed_states.map.jinja' import allowed_states %}
+{% if sls.split('.')[0] in allowed_states %}
+{%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
+{%   from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS, SO_MANAGED_INDICES %}
+{%   if GLOBALS.role != 'so-heavynode' %}
+{%     from 'elasticsearch/template.map.jinja' import ALL_ADDON_SETTINGS %}
+{%   endif %}
+
+escomponenttemplates:
+  file.recurse:
+    - name: /opt/so/conf/elasticsearch/templates/component
+    - source: salt://elasticsearch/templates/component
+    - user: 930
+    - group: 939
+    - clean: True
+    - onchanges_in:
+      - file: so-elasticsearch-templates-reload
+    - show_changes: False
+
+# Clean up legacy and non-SO managed templates from the elasticsearch/templates/index/ directory
+so_index_template_dir:
+  file.directory:
+    - name: /opt/so/conf/elasticsearch/templates/index
+    - clean: True
+    {%- if SO_MANAGED_INDICES %}
+    - require:
+      {%- for index in SO_MANAGED_INDICES %}
+      - file: so_index_template_{{index}}
+      {%- endfor %}
+    {%- endif %}
+
+# Auto-generate index templates for SO managed indices (directly defined in elasticsearch/defaults.yaml)
+#   These index templates are for the core SO datasets and are always required
+{%  for index, settings in ES_INDEX_SETTINGS.items() %}
+{%    if settings.index_template is defined %}
+so_index_template_{{index}}:
+  file.managed:
+    - name: /opt/so/conf/elasticsearch/templates/index/{{ index }}-template.json
+    - source: salt://elasticsearch/base-template.json.jinja
+    - defaults:
+        TEMPLATE_CONFIG: {{ settings.index_template }}
+    - template: jinja
+    - onchanges_in:
+      - file: so-elasticsearch-templates-reload
+{%    endif %}
+{%  endfor %}
+
+{%  if GLOBALS.role != "so-heavynode" %}
+# Auto-generate optional index templates for integration | input | content packages
+#   These index templates are not used by default (until user adds package to an agent policy).
+#   Pre-configured with standard defaults, and incorporated into SOC configuration for user customization.
+{%    for index,settings in ALL_ADDON_SETTINGS.items() %}
+{%      if settings.index_template is defined %}
+addon_index_template_{{index}}:
+  file.managed:
+    - name: /opt/so/conf/elasticsearch/templates/addon-index/{{ index }}-template.json
+    - source: salt://elasticsearch/base-template.json.jinja
+    - defaults:
+        TEMPLATE_CONFIG: {{ settings.index_template }}
+    - template: jinja
+    - show_changes: False
+    - onchanges_in:
+      - file: addon-elasticsearch-templates-reload
+{%       endif %}
+{%     endfor %}
+{%   endif %}
+
+{%  if GLOBALS.role in GLOBALS.manager_roles %}
+so-es-cluster-settings:
+  cmd.run:
+    - name: /usr/sbin/so-elasticsearch-cluster-settings
+    - cwd: /opt/so
+    - template: jinja
+    - require:
+      - docker_container: so-elasticsearch
+      - file: elasticsearch_sbin_jinja
+      - http: wait_for_so-elasticsearch
+{%  endif %}
+
+# heavynodes will only load ILM policies for SO managed indices. (Indicies defined in elasticsearch/defaults.yaml)
+so-elasticsearch-ilm-policy-load:
+  cmd.run:
+    - name: /usr/sbin/so-elasticsearch-ilm-policy-load
+    - cwd: /opt/so
+    - require:
+      - docker_container: so-elasticsearch
+      - file: so-elasticsearch-ilm-policy-load-script
+    - onchanges:
+      - file: so-elasticsearch-ilm-policy-load-script
+
+so-elasticsearch-templates-reload:
+  file.absent:
+    - name: /opt/so/state/estemplates.txt
+
+addon-elasticsearch-templates-reload:
+  file.absent:
+    - name: /opt/so/state/addon_estemplates.txt
+
+# so-elasticsearch-templates-load will have its first successful run during the 'so-elastic-fleet-setup' script
+so-elasticsearch-templates:
+  cmd.run:
+{%-  if GLOBALS.role == "so-heavynode" %}
+    - name: /usr/sbin/so-elasticsearch-templates-load --heavynode
+{%-  else %}
+    - name: /usr/sbin/so-elasticsearch-templates-load
+{%-  endif %}
+    - cwd: /opt/so
+    - template: jinja
+    - require:
+      - docker_container: so-elasticsearch
+      - file: elasticsearch_sbin_jinja
+
+so-elasticsearch-pipelines:
+  cmd.run:
+    - name: /usr/sbin/so-elasticsearch-pipelines {{ GLOBALS.hostname }}
+    - require:
+      - docker_container: so-elasticsearch
+      - file: so-elasticsearch-pipelines-script
+
+so-elasticsearch-roles-load:
+  cmd.run:
+    - name: /usr/sbin/so-elasticsearch-roles-load
+    - cwd: /opt/so
+    - template: jinja
+    - require:
+      - docker_container: so-elasticsearch
+      - file: elasticsearch_sbin_jinja
+
+{%  if grains.role in ['so-managersearch', 'so-manager', 'so-managerhype'] %}
+{%    set ap = "absent" %}
+{%  endif %}
+{%  if grains.role in ['so-eval', 'so-standalone', 'so-heavynode'] %}
+{%    if ELASTICSEARCHMERGED.index_clean %}
+{%      set ap = "present" %}
+{%    else %}
+{%      set ap = "absent" %}
+{%    endif %}
+{%  endif %}
+{%  if grains.role in ['so-eval', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-manager'] %}
+so-elasticsearch-indices-delete:
+  cron.{{ap}}:
+    - name: /usr/sbin/so-elasticsearch-indices-delete > /opt/so/log/elasticsearch/cron-elasticsearch-indices-delete.log 2>&1
+    - identifier: so-elasticsearch-indices-delete
+    - user: root
+    - minute: '*/5'
+    - hour: '*'
+    - daymonth: '*'
+    - month: '*'
+    - dayweek: '*'
+{%  endif %}
+
+{% else %}
+
+{{sls}}_state_not_allowed:
+  test.fail_without_changes:
+    - name: {{sls}}_state_not_allowed
+
+{% endif %}

salt/elasticsearch/config.sls

@@ -66,6 +66,8 @@ so-elasticsearch-ilm-policy-load-script:
     - group: 939
     - mode: 754
     - template: jinja
+    - defaults:
+        GLOBALS: {{ GLOBALS }}
     - show_changes: False
 
 so-elasticsearch-pipelines-script:
@@ -91,6 +93,13 @@ estemplatedir:
     - group: 939
     - makedirs: True
 
+esaddontemplatedir:
+  file.directory:
+    - name: /opt/so/conf/elasticsearch/templates/addon-index
+    - user: 930
+    - group: 939
+    - makedirs: True
+
 esrolesdir:
   file.directory:
     - name: /opt/so/conf/elasticsearch/roles

salt/elasticsearch/defaults.yaml

@@ -1,6 +1,6 @@
 elasticsearch:
   enabled: false
-  version: 9.0.8
+  version: 9.3.3
   index_clean: true
   vm:
     max_map_count: 1048576

salt/elasticsearch/enabled.sls

@@ -10,8 +10,6 @@
 {%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_NODES %}
 {%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_SEED_HOSTS %}
 {%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
-{%   set TEMPLATES = salt['pillar.get']('elasticsearch:templates', {}) %}
-{%   from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS %}
 
 include:
   - ca
@@ -19,6 +17,9 @@ include:
   - elasticsearch.ssl
   - elasticsearch.config
   - elasticsearch.sostatus
+{%- if GLOBALS.role != 'so-searchode' %}
+  - elasticsearch.cluster
+{%- endif%}
 
 so-elasticsearch:
   docker_container.running:
@@ -106,128 +107,19 @@ delete_so-elasticsearch_so-status.disabled:
     - name: /opt/so/conf/so-status/so-status.conf
     - regex: ^so-elasticsearch$
 
-{%   if GLOBALS.role != "so-searchnode" %}
-escomponenttemplates:
-  file.recurse:
-    - name: /opt/so/conf/elasticsearch/templates/component
-    - source: salt://elasticsearch/templates/component
-    - user: 930
-    - group: 939
-    - clean: True
-    - onchanges_in:
-      - file: so-elasticsearch-templates-reload
-    - show_changes: False
-      
-# Auto-generate templates from defaults file
-{%     for index, settings in ES_INDEX_SETTINGS.items() %}
- {%      if settings.index_template is defined %}
-es_index_template_{{index}}:
-  file.managed:
-    - name: /opt/so/conf/elasticsearch/templates/index/{{ index }}-template.json
-    - source: salt://elasticsearch/base-template.json.jinja
-    - defaults:
-      TEMPLATE_CONFIG: {{ settings.index_template }}
-    - template: jinja
-    - show_changes: False
-    - onchanges_in:
-      - file: so-elasticsearch-templates-reload
-{%       endif %}
-{%     endfor %}
-
-{%     if TEMPLATES %}
-# Sync custom templates to /opt/so/conf/elasticsearch/templates
-{%       for TEMPLATE in TEMPLATES %}
-es_template_{{TEMPLATE.split('.')[0] | replace("/","_") }}:
-  file.managed:
-    - source: salt://elasticsearch/templates/index/{{TEMPLATE}}
-{%         if 'jinja' in TEMPLATE.split('.')[-1] %}
-    - name: /opt/so/conf/elasticsearch/templates/index/{{TEMPLATE.split('/')[1] | replace(".jinja", "")}}
-    - template: jinja
-{%         else %}
-    - name: /opt/so/conf/elasticsearch/templates/index/{{TEMPLATE.split('/')[1]}}
-{%         endif %}
-    - user: 930
-    - group: 939
-    - show_changes: False
-    - onchanges_in:
-      - file: so-elasticsearch-templates-reload
-{%       endfor %}
-{%     endif %}
-
-{%     if GLOBALS.role in GLOBALS.manager_roles %}
-so-es-cluster-settings:
-  cmd.run:
-    - name: /usr/sbin/so-elasticsearch-cluster-settings
-    - cwd: /opt/so
-    - template: jinja
+wait_for_so-elasticsearch:
+  http.wait_for_successful_query:
+    - name: "https://localhost:9200/"
+    - username: 'so_elastic'
+    - password: '{{ ELASTICSEARCHMERGED.auth.users.so_elastic_user.pass }}'
+    - ssl: True
+    - verify_ssl: False
+    - status: 200
+    - wait_for: 300
+    - request_interval: 15
+    - backend: requests
     - require:
       - docker_container: so-elasticsearch
-      - file: elasticsearch_sbin_jinja
-{%     endif %}
-
-so-elasticsearch-ilm-policy-load:
-  cmd.run:
-    - name: /usr/sbin/so-elasticsearch-ilm-policy-load
-    - cwd: /opt/so
-    - require:
-      - docker_container: so-elasticsearch
-      - file: so-elasticsearch-ilm-policy-load-script
-    - onchanges:
-      - file: so-elasticsearch-ilm-policy-load-script
-
-so-elasticsearch-templates-reload:
-  file.absent:
-    - name: /opt/so/state/estemplates.txt
-
-so-elasticsearch-templates:
-  cmd.run:
-    - name: /usr/sbin/so-elasticsearch-templates-load
-    - cwd: /opt/so
-    - template: jinja
-    - require:
-      - docker_container: so-elasticsearch
-      - file: elasticsearch_sbin_jinja
-
-so-elasticsearch-pipelines:
-  cmd.run:
-    - name: /usr/sbin/so-elasticsearch-pipelines {{ GLOBALS.hostname }}
-    - require:
-      - docker_container: so-elasticsearch
-      - file: so-elasticsearch-pipelines-script
-
-so-elasticsearch-roles-load:
-  cmd.run:
-    - name: /usr/sbin/so-elasticsearch-roles-load
-    - cwd: /opt/so
-    - template: jinja
-    - require:
-      - docker_container: so-elasticsearch
-      - file: elasticsearch_sbin_jinja
-
-{%     if grains.role in ['so-managersearch', 'so-manager', 'so-managerhype'] %}
-{%       set ap = "absent" %}
-{%     endif %}
-{%     if grains.role in ['so-eval', 'so-standalone', 'so-heavynode'] %}
-{%       if ELASTICSEARCHMERGED.index_clean %}
-{%         set ap = "present" %}
-{%       else %}
-{%         set ap = "absent" %}
-{%       endif %}
-{%     endif %}  
-{%     if grains.role in ['so-eval', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-manager'] %}
-so-elasticsearch-indices-delete:
-  cron.{{ap}}:
-    - name: /usr/sbin/so-elasticsearch-indices-delete > /opt/so/log/elasticsearch/cron-elasticsearch-indices-delete.log 2>&1
-    - identifier: so-elasticsearch-indices-delete
-    - user: root
-    - minute: '*/5'
-    - hour: '*'
-    - daymonth: '*'
-    - month: '*'
-    - dayweek: '*'
-{%     endif %}
-
-{%   endif %}
 
 {% else %}
 

salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.2

@@ -10,24 +10,28 @@
   "processors": [
     {
       "set": {
+        "tag": "set_ecs_version_f5923549",
         "field": "ecs.version",
         "value": "8.17.0"
       }
     },
     {
       "set": {
+        "tag": "set_observer_vendor_ad9d35cc",
         "field": "observer.vendor",
         "value": "netgate"
       }
     },
     {
       "set": {
+        "tag": "set_observer_type_5dddf3ba",
         "field": "observer.type",
         "value": "firewall"
       }
     },
     {
       "rename": {
+        "tag": "rename_message_to_event_original_56a77271",
         "field": "message",
         "target_field": "event.original",
         "ignore_missing": true,
@@ -36,12 +40,14 @@
     },
     {
       "set": {
+        "tag": "set_event_kind_de80643c",
         "field": "event.kind",
         "value": "event"
       }
     },
     {
       "set": {
+        "tag": "set_event_timezone_4ca44cac",
         "field": "event.timezone",
         "value": "{{{_tmp.tz_offset}}}",
         "if": "ctx._tmp?.tz_offset != null && ctx._tmp?.tz_offset != 'local'"
@@ -49,6 +55,7 @@
     },
     {
       "grok": {
+        "tag": "grok_event_original_27d9c8c7",
         "description": "Parse syslog header",
         "field": "event.original",
         "patterns": [
@@ -72,6 +79,7 @@
     },
     {
       "date": {
+        "tag": "date__tmp_timestamp8601_to_timestamp_6ac9d3ce",
         "if": "ctx._tmp.timestamp8601 != null",
         "field": "_tmp.timestamp8601",
         "target_field": "@timestamp",
@@ -82,6 +90,7 @@
     },
     {
       "date": {
+        "tag": "date__tmp_timestamp_to_timestamp_f21e536e",
         "if": "ctx.event?.timezone != null && ctx._tmp?.timestamp != null",
         "field": "_tmp.timestamp",
         "target_field": "@timestamp",
@@ -95,6 +104,7 @@
     },
     {
       "grok": {
+        "tag": "grok_process_name_cef3d489",
         "description": "Set Event Provider",
         "field": "process.name",
         "patterns": [
@@ -107,71 +117,83 @@
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.23.1-firewall",
+        "tag": "pipeline_e16851a7",
+        "name": "logs-pfsense.log-1.25.2-firewall",
         "if": "ctx.event.provider == 'filterlog'"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.23.1-openvpn",
+        "tag": "pipeline_828590b5",
+        "name": "logs-pfsense.log-1.25.2-openvpn",
         "if": "ctx.event.provider == 'openvpn'"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.23.1-ipsec",
+        "tag": "pipeline_9d37039c",
+        "name": "logs-pfsense.log-1.25.2-ipsec",
         "if": "ctx.event.provider == 'charon'"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.23.1-dhcp",
-        "if": "[\"dhcpd\", \"dhclient\", \"dhcp6c\"].contains(ctx.event.provider)"
+        "tag": "pipeline_ad56bbca",
+        "name": "logs-pfsense.log-1.25.2-dhcp",
+        "if": "[\"dhcpd\", \"dhclient\", \"dhcp6c\", \"dnsmasq-dhcp\"].contains(ctx.event.provider)"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.23.1-unbound",
+        "tag": "pipeline_dd85553d",
+        "name": "logs-pfsense.log-1.25.2-unbound",
         "if": "ctx.event.provider == 'unbound'"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.23.1-haproxy",
+        "tag": "pipeline_720ed255",
+        "name": "logs-pfsense.log-1.25.2-haproxy",
         "if": "ctx.event.provider == 'haproxy'"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.23.1-php-fpm",
+        "tag": "pipeline_456beba5",
+        "name": "logs-pfsense.log-1.25.2-php-fpm",
         "if": "ctx.event.provider == 'php-fpm'"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.23.1-squid",
+        "tag": "pipeline_a0d89375",
+        "name": "logs-pfsense.log-1.25.2-squid",
         "if": "ctx.event.provider == 'squid'"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.23.1-snort",
+        "tag": "pipeline_c2f1ed55",
+        "name": "logs-pfsense.log-1.25.2-snort",
         "if": "ctx.event.provider == 'snort'"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.23.1-suricata",
+        "tag":"pipeline_33db1c9e",
+        "name": "logs-pfsense.log-1.25.2-suricata",
         "if": "ctx.event.provider == 'suricata'"
       }
     },
     {
       "drop": {
-        "if": "![\"filterlog\", \"openvpn\", \"charon\", \"dhcpd\", \"dhclient\", \"dhcp6c\", \"unbound\", \"haproxy\", \"php-fpm\", \"squid\", \"snort\", \"suricata\"].contains(ctx.event?.provider)"
+        "tag": "drop_9d7c46f8",
+        "if": "![\"filterlog\", \"openvpn\", \"charon\", \"dhcpd\", \"dnsmasq-dhcp\", \"dhclient\", \"dhcp6c\", \"unbound\", \"haproxy\", \"php-fpm\", \"squid\", \"snort\", \"suricata\"].contains(ctx.event?.provider)"
       }
     },
     {
       "append": {
+        "tag": "append_event_category_4780a983",
         "field": "event.category",
         "value": "network",
         "if": "ctx.network != null"
@@ -179,6 +201,7 @@
     },
     {
       "convert": {
+        "tag": "convert_source_address_to_source_ip_f5632a20",
         "field": "source.address",
         "target_field": "source.ip",
         "type": "ip",
@@ -188,6 +211,7 @@
     },
     {
       "convert": {
+        "tag": "convert_destination_address_to_destination_ip_f1388f0c",
         "field": "destination.address",
         "target_field": "destination.ip",
         "type": "ip",
@@ -197,6 +221,7 @@
     },
     {
       "set": {
+        "tag": "set_network_type_1f1d940a",
         "field": "network.type",
         "value": "ipv6",
         "if": "ctx.source?.ip != null && ctx.source.ip.contains(\":\")"
@@ -204,6 +229,7 @@
     },
     {
       "set": {
+        "tag": "set_network_type_69deca38",
         "field": "network.type",
         "value": "ipv4",
         "if": "ctx.source?.ip != null && ctx.source.ip.contains(\".\")"
@@ -211,6 +237,7 @@
     },
     {
       "geoip": {
+        "tag": "geoip_source_ip_to_source_geo_da2e41b2",
         "field": "source.ip",
         "target_field": "source.geo",
         "ignore_missing": true
@@ -218,6 +245,7 @@
     },
     {
       "geoip": {
+        "tag": "geoip_destination_ip_to_destination_geo_ab5e2968",
         "field": "destination.ip",
         "target_field": "destination.geo",
         "ignore_missing": true
@@ -225,6 +253,7 @@
     },
     {
       "geoip": {
+        "tag": "geoip_source_ip_to_source_as_28d69883",
         "ignore_missing": true,
         "database_file": "GeoLite2-ASN.mmdb",
         "field": "source.ip",
@@ -237,6 +266,7 @@
     },
     {
       "geoip": {
+        "tag": "geoip_destination_ip_to_destination_as_8a007787",
         "database_file": "GeoLite2-ASN.mmdb",
         "field": "destination.ip",
         "target_field": "destination.as",
@@ -249,6 +279,7 @@
     },
     {
       "rename": {
+        "tag": "rename_source_as_asn_to_source_as_number_a917047d",
         "field": "source.as.asn",
         "target_field": "source.as.number",
         "ignore_missing": true
@@ -256,6 +287,7 @@
     },
     {
       "rename": {
+        "tag": "rename_source_as_organization_name_to_source_as_organization_name_f1362d0b",
         "field": "source.as.organization_name",
         "target_field": "source.as.organization.name",
         "ignore_missing": true
@@ -263,6 +295,7 @@
     },
     {
       "rename": {
+        "tag": "rename_destination_as_asn_to_destination_as_number_3b459fcd",
         "field": "destination.as.asn",
         "target_field": "destination.as.number",
         "ignore_missing": true
@@ -270,6 +303,7 @@
     },
     {
       "rename": {
+        "tag": "rename_destination_as_organization_name_to_destination_as_organization_name_814bd459",
         "field": "destination.as.organization_name",
         "target_field": "destination.as.organization.name",
         "ignore_missing": true
@@ -277,12 +311,14 @@
     },
     {
       "community_id": {
+        "tag": "community_id_d2308e7a",
         "target_field": "network.community_id",
         "ignore_failure": true
       }
     },
     {
       "grok": {
+        "tag": "grok_observer_ingress_interface_name_968018d3",
         "field": "observer.ingress.interface.name",
         "patterns": [
           "%{DATA}.%{NONNEGINT:observer.ingress.vlan.id}"
@@ -293,6 +329,7 @@
     },
     {
       "set": {
+        "tag": "set_network_vlan_id_efd4d96a",
         "field": "network.vlan.id",
         "copy_from": "observer.ingress.vlan.id",
         "ignore_empty_value": true
@@ -300,6 +337,7 @@
     },
     {
       "append": {
+        "tag": "append_related_ip_c1a6356b",
         "field": "related.ip",
         "value": "{{{destination.ip}}}",
         "allow_duplicates": false,
@@ -308,6 +346,7 @@
     },
     {
       "append": {
+        "tag": "append_related_ip_8121c591",
         "field": "related.ip",
         "value": "{{{source.ip}}}",
         "allow_duplicates": false,
@@ -316,6 +355,7 @@
     },
     {
       "append": {
+        "tag": "append_related_ip_53b62ed8",
         "field": "related.ip",
         "value": "{{{source.nat.ip}}}",
         "allow_duplicates": false,
@@ -324,6 +364,7 @@
     },
     {
       "append": {
+        "tag": "append_related_hosts_6f162628",
         "field": "related.hosts",
         "value": "{{{destination.domain}}}",
         "if": "ctx.destination?.domain != null"
@@ -331,6 +372,7 @@
     },
     {
       "append": {
+        "tag": "append_related_user_c036eec2",
         "field": "related.user",
         "value": "{{{user.name}}}",
         "if": "ctx.user?.name != null"
@@ -338,6 +380,7 @@
     },
     {
       "set": {
+        "tag": "set_network_direction_cb1e3125",
         "field": "network.direction",
         "value": "{{{network.direction}}}bound",
         "if": "ctx.network?.direction != null && ctx.network?.direction =~ /^(in|out)$/"
@@ -345,6 +388,7 @@
     },
     {
       "remove": {
+        "tag": "remove_a82e20f2",
         "field": [
           "_tmp"
         ],
@@ -353,11 +397,21 @@
     },
     {
       "script": {
+        "tag": "script_a7f2c062",
         "lang": "painless",
         "description": "This script processor iterates over the whole document to remove fields with null values.",
         "source": "void handleMap(Map map) {\n  for (def x : map.values()) {\n    if (x instanceof Map) {\n        handleMap(x);\n    } else if (x instanceof List) {\n        handleList(x);\n    }\n  }\n  map.values().removeIf(v -> v == null || (v instanceof String && v == \"-\"));\n}\nvoid handleList(List list) {\n  for (def x : list) {\n      if (x instanceof Map) {\n          handleMap(x);\n      } else if (x instanceof List) {\n          handleList(x);\n      }\n  }\n}\nhandleMap(ctx);\n"
       }
     },
+    {
+      "append": {
+        "tag": "append_preserve_original_event_on_error",
+        "field": "tags",
+        "value": "preserve_original_event",
+        "allow_duplicates": false,
+        "if": "ctx.error?.message != null"
+      }
+    },
     {
       "pipeline": {
         "name": "global@custom",
@@ -405,7 +459,14 @@
     {
       "append": {
         "field": "error.message",
-        "value": "{{{ _ingest.on_failure_message }}}"
+        "value": "Processor '{{{ _ingest.on_failure_processor_type }}}' {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}'"
+      }
+    },
+    {
+      "append": {
+        "field": "tags",
+        "value": "preserve_original_event",
+        "allow_duplicates": false
       }
     }
   ]

salt/elasticsearch/files/ingest/suricata.common

@@ -22,6 +22,12 @@
                 "ignore_failure": true
             }
         },
+        {
+            "lowercase": {
+                "field": "network.transport",
+                "ignore_failure": true
+            }
+        },
         {
             "rename": {
                 "field": "message2.in_iface",

salt/elasticsearch/files/log4j2.properties

@@ -45,3 +45,7 @@ appender.rolling_json.strategy.action.condition.nested_condition.age = 1D
 rootLogger.level = info
 rootLogger.appenderRef.rolling.ref = rolling
 rootLogger.appenderRef.rolling_json.ref = rolling_json
+
+# Suppress NotEntitledException WARNs (ES 9.3.3 bug)
+logger.entitlement_security.name = org.elasticsearch.entitlement.runtime.policy.PolicyManager.x-pack-security.org.elasticsearch.security.org.elasticsearch.xpack.security
+logger.entitlement_security.level = error

salt/elasticsearch/template.map.jinja

@@ -14,16 +14,43 @@
 
 {% set ES_INDEX_SETTINGS_ORIG = ELASTICSEARCHDEFAULTS.elasticsearch.index_settings %}
 
+{% set ALL_ADDON_INTEGRATION_DEFAULTS = {} %}
+{% set ALL_ADDON_SETTINGS_ORIG = {} %}
+{% set ALL_ADDON_SETTINGS_GLOBAL_OVERRIDES = {} %}
+{% set ALL_ADDON_SETTINGS = {} %}
 {# start generation of integration default index_settings #}
-{% if salt['file.file_exists']('/opt/so/state/esfleet_package_components.json') and salt['file.file_exists']('/opt/so/state/esfleet_component_templates.json') %}
-{%   set check_package_components = salt['file.stats']('/opt/so/state/esfleet_package_components.json') %}
-{%   if check_package_components.size > 1 %}
+{% if salt['file.file_exists']('/opt/so/state/esfleet_component_templates.json') %}
+{#   import integration type defaults #}
+{%   if salt['file.file_exists']('/opt/so/state/esfleet_package_components.json') %}
+{%     set check_integration_package_components = salt['file.stats']('/opt/so/state/esfleet_package_components.json') %}
+{%     if check_integration_package_components.size > 1 %}
 {%       from 'elasticfleet/integration-defaults.map.jinja' import ADDON_INTEGRATION_DEFAULTS %}
-{%    for index, settings in ADDON_INTEGRATION_DEFAULTS.items() %}
-{%      do ES_INDEX_SETTINGS_ORIG.update({index: settings}) %}
-{%    endfor %}
+{%       do ALL_ADDON_INTEGRATION_DEFAULTS.update(ADDON_INTEGRATION_DEFAULTS) %}
 {%     endif %}
 {%   endif %}
+
+{#   import input type defaults #}
+{%   if salt['file.file_exists']('/opt/so/state/esfleet_input_package_components.json') %}
+{%     set check_input_package_components = salt['file.stats']('/opt/so/state/esfleet_input_package_components.json') %}
+{%     if check_input_package_components.size > 1 %}
+{%       from 'elasticfleet/input-defaults.map.jinja' import ADDON_INPUT_INTEGRATION_DEFAULTS %}
+{%       do ALL_ADDON_INTEGRATION_DEFAULTS.update(ADDON_INPUT_INTEGRATION_DEFAULTS) %}
+{%     endif %}
+{%   endif %}
+
+{#   import content type defaults #}
+{%   if salt['file.file_exists']('/opt/so/state/esfleet_content_package_components.json') %}
+{%     set check_content_package_components = salt['file.stats']('/opt/so/state/esfleet_content_package_components.json') %}
+{%     if check_content_package_components.size > 1 %}
+{%       from 'elasticfleet/content-defaults.map.jinja' import ADDON_CONTENT_INTEGRATION_DEFAULTS %}
+{%       do ALL_ADDON_INTEGRATION_DEFAULTS.update(ADDON_CONTENT_INTEGRATION_DEFAULTS) %}
+{%     endif %}
+{%   endif %}
+
+{%   for index, settings in ALL_ADDON_INTEGRATION_DEFAULTS.items() %}
+{%     do ALL_ADDON_SETTINGS_ORIG.update({index: settings}) %}
+{%   endfor %}
+{% endif %}
 {# end generation of integration default index_settings #}
 
 {% set ES_INDEX_SETTINGS_GLOBAL_OVERRIDES = {} %}
@@ -31,25 +58,33 @@
 {%   do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.update({index: salt['defaults.merge'](ELASTICSEARCHDEFAULTS.elasticsearch.index_settings[index], PILLAR_GLOBAL_OVERRIDES, in_place=False)}) %}
 {% endfor %}
 
+{% if ALL_ADDON_SETTINGS_ORIG.keys() | length > 0 %}
+{%   for index in ALL_ADDON_SETTINGS_ORIG.keys() %}
+{%     do ALL_ADDON_SETTINGS_GLOBAL_OVERRIDES.update({index: salt['defaults.merge'](ALL_ADDON_SETTINGS_ORIG[index], PILLAR_GLOBAL_OVERRIDES, in_place=False)}) %}
+{%   endfor %}
+{% endif %}
+
 {% set ES_INDEX_SETTINGS = {} %}
-{% do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.update(salt['defaults.merge'](ES_INDEX_SETTINGS_GLOBAL_OVERRIDES, ES_INDEX_PILLAR, in_place=False)) %}
-{% for index, settings in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.items() %}
+{% macro create_final_index_template(DEFINED_SETTINGS, GLOBAL_OVERRIDES, FINAL_INDEX_SETTINGS) %}
+
+{% do GLOBAL_OVERRIDES.update(salt['defaults.merge'](GLOBAL_OVERRIDES, ES_INDEX_PILLAR, in_place=False)) %}
+{% for index, settings in GLOBAL_OVERRIDES.items() %}
 
 {#   prevent this action from being performed on custom defined indices. #}
 {#   the custom defined index is not present in either of the dictionaries and fails to reder. #}
-{%   if index in ES_INDEX_SETTINGS_ORIG and index in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES %}
+{%   if index in DEFINED_SETTINGS and index in GLOBAL_OVERRIDES %}
 
 {#     dont merge policy from the global_overrides if policy isn't defined in the original index settingss #}
 {#     this will prevent so-elasticsearch-ilm-policy-load from trying to load policy on non ILM manged indices #}
-{%     if not ES_INDEX_SETTINGS_ORIG[index].policy is defined and ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy is defined %}
-{%       do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].pop('policy') %}
+{%     if not DEFINED_SETTINGS[index].policy is defined and GLOBAL_OVERRIDES[index].policy is defined %}
+{%       do GLOBAL_OVERRIDES[index].pop('policy') %}
 {%     endif %}
 
 {#     this prevents and index from inderiting a policy phase from global overrides if it wasnt defined in the defaults. #}
-{%     if ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy is defined %}
-{%       for phase in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy.phases.copy() %}
-{%         if ES_INDEX_SETTINGS_ORIG[index].policy.phases[phase] is not defined %}
-{%           do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy.phases.pop(phase) %}
+{%     if GLOBAL_OVERRIDES[index].policy is defined %}
+{%       for phase in GLOBAL_OVERRIDES[index].policy.phases.copy() %}
+{%         if DEFINED_SETTINGS[index].policy.phases[phase] is not defined %}
+{%           do GLOBAL_OVERRIDES[index].policy.phases.pop(phase) %}
 {%         endif %}
 {%       endfor %}
 {%     endif %}
@@ -111,5 +146,14 @@
 {%     endfor %}
 {%   endif %}
 
-{%   do ES_INDEX_SETTINGS.update({index | replace("_x_", "."): ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index]}) %}
+{%   do FINAL_INDEX_SETTINGS.update({index | replace("_x_", "."): GLOBAL_OVERRIDES[index]}) %}
+{% endfor %}
+{% endmacro %}
+
+{{ create_final_index_template(ES_INDEX_SETTINGS_ORIG, ES_INDEX_SETTINGS_GLOBAL_OVERRIDES, ES_INDEX_SETTINGS) }}
+{{ create_final_index_template(ALL_ADDON_SETTINGS_ORIG, ALL_ADDON_SETTINGS_GLOBAL_OVERRIDES, ALL_ADDON_SETTINGS) }}
+
+{% set SO_MANAGED_INDICES = [] %}
+{% for index, settings in ES_INDEX_SETTINGS.items() %}
+{%   do SO_MANAGED_INDICES.append(index) %}
 {% endfor %}

salt/elasticsearch/tools/sbin/so-elasticsearch-component-templates-list

@@ -6,8 +6,19 @@
 # Elastic License 2.0.
 
 . /usr/sbin/so-common
-if [ "$1" == "" ]; then
-    curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_component_template | jq '.component_templates[] |.name'| sort
+
+if [[ -z "$1" ]]; then
+    if output=$(so-elasticsearch-query "_component_template" --retry 3 --retry-delay 1 --fail); then
+        jq '[.component_templates[] | .name] | sort' <<< "$output"
     else
-    curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_component_template/$1 | jq
+        echo "Failed to retrieve component templates from Elasticsearch."
+        exit 1
+    fi
+else
+    if output=$(so-elasticsearch-query "_component_template/$1" --retry 3 --retry-delay 1 --fail); then
+        jq <<< "$output"
+    else
+        echo "Failed to retrieve component template '$1' from Elasticsearch."
+        exit 1
+    fi
 fi

salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load

@@ -0,0 +1,253 @@
+#!/bin/bash
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+. /usr/sbin/so-common
+
+SO_STATEFILE_SUCCESS=/opt/so/state/estemplates.txt
+ADDON_STATEFILE_SUCCESS=/opt/so/state/addon_estemplates.txt
+ELASTICSEARCH_TEMPLATES_DIR="/opt/so/conf/elasticsearch/templates"
+SO_TEMPLATES_DIR="${ELASTICSEARCH_TEMPLATES_DIR}/index"
+ADDON_TEMPLATES_DIR="${ELASTICSEARCH_TEMPLATES_DIR}/addon-index"
+SO_LOAD_FAILURES=0
+ADDON_LOAD_FAILURES=0
+SO_LOAD_FAILURES_NAMES=()
+ADDON_LOAD_FAILURES_NAMES=()
+IS_HEAVYNODE="false"
+FORCE="false"
+VERBOSE="false"
+SHOULD_EXIT_ON_FAILURE="true"
+
+# If soup is running, ignore errors
+pgrep soup >/dev/null && SHOULD_EXIT_ON_FAILURE="false"
+
+while [[ $# -gt 0 ]]; do
+    case "$1" in
+    --heavynode)
+        IS_HEAVYNODE="true"
+        ;;
+    --force)
+        FORCE="true"
+        ;;
+    --verbose)
+        VERBOSE="true"
+        ;;
+    *)
+        echo "Usage: $0 [options]"
+        echo "Options:"
+        echo "  --heavynode     Only loads index templates specific to heavynodes"
+        echo "  --force     Force reload all templates regardless of statefiles (default: false)"
+        echo "  --verbose     Enable verbose output"
+        exit 1
+        ;;
+    esac
+    shift
+done
+
+load_template() {
+    local uri="$1"
+    local file="$2"
+
+    echo "Loading template file $file"
+    if ! output=$(retry 3 3 "so-elasticsearch-query $uri -d@$file -XPUT" "{\"acknowledged\":true}"); then
+        echo "$output"
+
+        return 1
+
+    elif [[ "$VERBOSE" == "true" ]]; then
+        echo "$output"
+    fi
+
+}
+
+check_required_component_template_exists() {
+    local required
+    local missing
+    local file=$1
+
+    required=$(jq '[((.composed_of //[]) - (.ignore_missing_component_templates // []))[]]' "$file")
+    missing=$(jq -n --argjson required "$required" --argjson component_templates "$component_templates" '(($required) - ($component_templates))')
+
+    if [[ $(jq length <<<"$missing") -gt 0 ]]; then
+
+        return 1
+    fi
+}
+
+check_heavynode_compatiable_index_template() {
+    # The only templates that are relevant to heavynodes are from datasets defined in elasticagent/files/elastic-agent.yml.jinja.
+    # Heavynodes do not have fleet server packages installed and do not support elastic agents reporting directly to them.
+    local -A heavynode_index_templates=(
+        ["so-import"]=1
+        ["so-syslog"]=1
+        ["so-logs-soc"]=1
+        ["so-suricata"]=1
+        ["so-suricata.alerts"]=1
+        ["so-zeek"]=1
+        ["so-strelka"]=1
+    )
+
+    local template_name="$1"
+
+    if [[ ! -v heavynode_index_templates["$template_name"] ]]; then
+
+        return 1
+    fi
+
+}
+
+load_component_templates() {
+    local printed_name="$1"
+    local pattern="${ELASTICSEARCH_TEMPLATES_DIR}/component/$2"
+    local append_mappings="${3:-"false"}"
+
+    # current state of nullglob shell option
+    shopt -q nullglob && nullglob_set=1 || nullglob_set=0
+
+    shopt -s nullglob
+    echo -e "\nLoading $printed_name component templates...\n"
+    for component in "$pattern"/*.json; do
+        tmpl_name=$(basename "${component%.json}")
+
+        if [[ "$append_mappings" == "true" ]]; then
+            # avoid duplicating "-mappings" if it already exists in the component template filename
+            tmpl_name="${tmpl_name%-mappings}-mappings"
+        fi
+
+        if ! load_template "_component_template/${tmpl_name}" "$component"; then
+            SO_LOAD_FAILURES=$((SO_LOAD_FAILURES + 1))
+            SO_LOAD_FAILURES_NAMES+=("$component")
+        fi
+    done
+
+    # restore nullglob shell option if needed
+    if [[ $nullglob_set -eq 1 ]]; then
+        shopt -u nullglob
+    fi
+}
+
+check_elasticsearch_responsive() {
+    # Cannot load templates if Elasticsearch is not responding.
+    #  NOTE: Slightly faster exit w/ failure than previous "retry 240 1" if there is a problem with Elasticsearch the
+    #    script should exit sooner rather than hang at the 'so-elasticsearch-templates' salt state.
+    retry 3 15 "so-elasticsearch-query / --output /dev/null --fail" ||
+        fail "Elasticsearch is not responding. Please review Elasticsearch logs /opt/so/log/elasticsearch/securityonion.log for more details. Additionally, consider running so-elasticsearch-troubleshoot."
+}
+
+if [[ "$FORCE" == "true" || ! -f "$SO_STATEFILE_SUCCESS" ]]; then
+    check_elasticsearch_responsive
+
+    if [[ "$IS_HEAVYNODE" == "false" ]]; then
+        # TODO: Better way to check if fleet server is installed vs checking for Elastic Defend component template.
+        fleet_check="logs-endpoint.alerts@package"
+        if ! so-elasticsearch-query "_component_template/$fleet_check" --output /dev/null --retry 5 --retry-delay 3 --fail; then
+            # This check prevents so-elasticsearch-templates-load from running before so-elastic-fleet-setup has run.
+            echo -e "\nPackage $fleet_check not yet installed. Fleet Server may not be fully configured yet."
+            # Fleet Server is required because some SO index templates depend on components installed via
+            #  specific integrations eg Elastic Defend. These are components that we do not manually create / manage
+            #  via /opt/so/saltstack/salt/elasticsearch/templates/component/
+
+            exit 0
+        fi
+    fi
+
+    # load_component_templates "Name" "directory" "append '-mappings'?"
+    load_component_templates "ECS" "ecs" "true"
+    load_component_templates "Elastic Agent" "elastic-agent"
+    load_component_templates "Security Onion" "so"
+
+    component_templates=$(so-elasticsearch-component-templates-list)
+    echo -e "Loading Security Onion index templates...\n"
+    for so_idx_tmpl in "${SO_TEMPLATES_DIR}"/*.json; do
+        tmpl_name=$(basename "${so_idx_tmpl%-template.json}")
+
+        if [[ "$IS_HEAVYNODE" == "true" ]]; then
+            # TODO: Better way to load only heavynode specific templates
+            if ! check_heavynode_compatiable_index_template "$tmpl_name"; then
+                if [[ "$VERBOSE" == "true" ]]; then
+                    echo "Skipping over $so_idx_tmpl, template is not a heavynode specific index template."
+                fi
+
+                continue
+            fi
+        fi
+
+        if check_required_component_template_exists "$so_idx_tmpl"; then
+            if ! load_template "_index_template/$tmpl_name" "$so_idx_tmpl"; then
+                SO_LOAD_FAILURES=$((SO_LOAD_FAILURES + 1))
+                SO_LOAD_FAILURES_NAMES+=("$so_idx_tmpl")
+            fi
+        else
+            echo "Skipping over $so_idx_tmpl due to missing required component template(s)."
+            SO_LOAD_FAILURES=$((SO_LOAD_FAILURES + 1))
+            SO_LOAD_FAILURES_NAMES+=("$so_idx_tmpl")
+
+            continue
+        fi
+    done
+
+    if [[ $SO_LOAD_FAILURES -eq 0 ]]; then
+        echo "All Security Onion core templates loaded successfully."
+
+        touch "$SO_STATEFILE_SUCCESS"
+    else
+        echo "Encountered $SO_LOAD_FAILURES failure(s) loading templates:"
+        for failed_template in "${SO_LOAD_FAILURES_NAMES[@]}"; do
+            echo "  - $failed_template"
+        done
+        if [[ "$SHOULD_EXIT_ON_FAILURE" == "true" ]]; then
+            fail "Failed to load all Security Onion core templates successfully."
+        fi
+    fi
+else
+
+    echo "Security Onion core templates already loaded"
+fi
+
+# Start loading addon templates
+if [[ (-d "$ADDON_TEMPLATES_DIR" && -f "$SO_STATEFILE_SUCCESS" && "$IS_HEAVYNODE" == "false" && ! -f "$ADDON_STATEFILE_SUCCESS") || (-d "$ADDON_TEMPLATES_DIR" && "$IS_HEAVYNODE" == "false" && "$FORCE" == "true") ]]; then
+
+    check_elasticsearch_responsive
+
+    echo -e "\nLoading addon integration index templates...\n"
+    component_templates=$(so-elasticsearch-component-templates-list)
+
+    for addon_idx_tmpl in "${ADDON_TEMPLATES_DIR}"/*.json; do
+        tmpl_name=$(basename "${addon_idx_tmpl%-template.json}")
+
+        if check_required_component_template_exists "$addon_idx_tmpl"; then
+            if ! load_template "_index_template/${tmpl_name}" "$addon_idx_tmpl"; then
+                ADDON_LOAD_FAILURES=$((ADDON_LOAD_FAILURES + 1))
+                ADDON_LOAD_FAILURES_NAMES+=("$addon_idx_tmpl")
+            fi
+        else
+            echo "Skipping over $addon_idx_tmpl due to missing required component template(s)."
+            ADDON_LOAD_FAILURES=$((ADDON_LOAD_FAILURES + 1))
+            ADDON_LOAD_FAILURES_NAMES+=("$addon_idx_tmpl")
+
+            continue
+        fi
+    done
+
+    if [[ $ADDON_LOAD_FAILURES -eq 0 ]]; then
+        echo "All addon integration templates loaded successfully."
+
+        touch "$ADDON_STATEFILE_SUCCESS"
+    else
+        echo "Encountered $ADDON_LOAD_FAILURES failure(s) loading addon integration templates:"
+        for failed_template in "${ADDON_LOAD_FAILURES_NAMES[@]}"; do
+            echo "  - $failed_template"
+        done
+        if [[ "$SHOULD_EXIT_ON_FAILURE" == "true" ]]; then
+            fail "Failed to load all addon integration templates successfully."
+        fi
+    fi
+
+elif [[ ! -f "$SO_STATEFILE_SUCCESS" && "$IS_HEAVYNODE" == "false" ]]; then
+    echo "Skipping loading addon integration templates until Security Onion core templates have been loaded."
+
+elif [[ -f "$ADDON_STATEFILE_SUCCESS" && "$IS_HEAVYNODE" == "false" && "$FORCE" == "false" ]]; then
+    echo "Addon integration templates already loaded"
+fi

salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load

@@ -7,6 +7,9 @@
 . /usr/sbin/so-common
 
 {%- from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS %}
+{%- if GLOBALS.role != "so-heavynode" %}
+{%-   from 'elasticsearch/template.map.jinja' import ALL_ADDON_SETTINGS %}
+{%- endif %}
 
 {%- for index, settings in ES_INDEX_SETTINGS.items() %}
 {%-   if settings.policy is defined %}
@@ -33,3 +36,13 @@
 {%-   endif %}
 {%- endfor %}
 echo
+{%- if GLOBALS.role != "so-heavynode" %}
+{%-   for index, settings in ALL_ADDON_SETTINGS.items() %}
+{%-     if settings.policy is defined %}
+  echo
+  echo "Setting up {{ index }}-logs policy..."
+  curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://localhost:9200/_ilm/policy/{{ index }}-logs" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }'
+  echo
+{%-     endif %}
+{%-   endfor %}
+{%- endif %}

salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load

@@ -1,165 +0,0 @@
-#!/bin/bash
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-{%- import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %}
-{%  from 'vars/globals.map.jinja' import GLOBALS %}
-
-STATE_FILE_INITIAL=/opt/so/state/estemplates_initial_load_attempt.txt
-STATE_FILE_SUCCESS=/opt/so/state/estemplates.txt
-
-if [[ -f $STATE_FILE_INITIAL ]]; then
-  # The initial template load has already run. As this is a subsequent load, all dependencies should 
-  # already be satisified. Therefore, immediately exit/abort this script upon any template load failure
-  # since this is an unrecoverable failure.
-  should_exit_on_failure=1
-else
-  # This is the initial template load, and there likely are some components not yet setup in Elasticsearch.
-  # Therefore load as many templates as possible at this time and if an error occurs proceed to the next 
-  # template. But if at least one template fails to load do not mark the templates as having been loaded.
-  # This will allow the next load to resume the load of the templates that failed to load initially.
-  should_exit_on_failure=0
-  echo "This is the initial template load"
-fi
-
-# If soup is running, ignore errors
-pgrep soup > /dev/null && should_exit_on_failure=0
-
-load_failures=0
-
-load_template() {
-  uri=$1
-  file=$2
-
-  echo "Loading template file $i" 
-  if ! retry 3 1 "so-elasticsearch-query $uri -d@$file -XPUT" "{\"acknowledged\":true}"; then
-    if [[ $should_exit_on_failure -eq 1 ]]; then
-      fail "Could not load template file: $file"
-    else
-      load_failures=$((load_failures+1))
-      echo "Incremented load failure counter: $load_failures"
-    fi
-  fi
-}
-
-if [ ! -f $STATE_FILE_SUCCESS ]; then
-  echo "State file $STATE_FILE_SUCCESS not found. Running so-elasticsearch-templates-load."
-
-  . /usr/sbin/so-common
-
-  {% if GLOBALS.role != 'so-heavynode' %}
-  if [ -f /usr/sbin/so-elastic-fleet-common ]; then
-    . /usr/sbin/so-elastic-fleet-common
-  fi
-  {% endif %}
-
-  default_conf_dir=/opt/so/conf
-
-  # Define a default directory to load pipelines from
-  ELASTICSEARCH_TEMPLATES="$default_conf_dir/elasticsearch/templates/"
-
-  {% if GLOBALS.role == 'so-heavynode' %}
-  file="/opt/so/conf/elasticsearch/templates/index/so-common-template.json"
-  {% else %}
-  file="/usr/sbin/so-elastic-fleet-common"
-  {% endif %}
-
-  if [ -f "$file" ]; then
-    # Wait for ElasticSearch to initialize
-    echo -n "Waiting for ElasticSearch..."
-    retry 240 1 "so-elasticsearch-query / -k --output /dev/null --silent --head --fail" || fail "Connection attempt timed out.  Unable to connect to ElasticSearch.  \nPlease try: \n  -checking log(s) in /var/log/elasticsearch/\n  -running 'sudo docker ps' \n  -running 'sudo so-elastic-restart'"
-    {% if GLOBALS.role != 'so-heavynode' %}
-    TEMPLATE="logs-endpoint.alerts@package"
-    INSTALLED=$(so-elasticsearch-query _component_template/$TEMPLATE | jq -r .component_templates[0].name)
-    if [ "$INSTALLED" != "$TEMPLATE" ]; then
-      echo
-      echo "Packages not yet installed."
-      echo
-      exit 0
-    fi
-    {% endif %}
-
-    touch $STATE_FILE_INITIAL
-
-    cd ${ELASTICSEARCH_TEMPLATES}/component/ecs
-
-    echo "Loading ECS component templates..."
-    for i in *; do 
-      TEMPLATE=$(echo $i | cut -d '.' -f1) 
-      load_template "_component_template/${TEMPLATE}-mappings" "$i"
-    done
-    echo
-
-    cd ${ELASTICSEARCH_TEMPLATES}/component/elastic-agent
-
-    echo "Loading Elastic Agent component templates..."
-    {% if GLOBALS.role == 'so-heavynode' %}
-    component_pattern="so-*"
-    {% else %}
-    component_pattern="*"
-    {% endif %}
-    for i in $component_pattern; do 
-      TEMPLATE=${i::-5} 
-      load_template "_component_template/$TEMPLATE" "$i"
-    done
-    echo
-	    
-    # Load SO-specific component templates
-    cd ${ELASTICSEARCH_TEMPLATES}/component/so
-
-    echo "Loading Security Onion component templates..."
-    for i in *; do 
-      TEMPLATE=$(echo $i | cut -d '.' -f1); 
-      load_template "_component_template/$TEMPLATE" "$i"
-    done
-    echo
-
-    # Load SO index templates
-    cd ${ELASTICSEARCH_TEMPLATES}/index
-
-    echo "Loading Security Onion index templates..."
-    shopt -s extglob
-    {% if GLOBALS.role == 'so-heavynode' %}
-    pattern="!(*1password*|*aws*|*azure*|*cloudflare*|*elastic_agent*|*fim*|*github*|*google*|*osquery*|*system*|*windows*|*endpoint*|*elasticsearch*|*generic*|*fleet_server*|*soc*)"
-    {% else %}
-    pattern="*"
-    {% endif %}
-    # Index templates will be skipped if the following conditions are met:
-    # 1. The template is part of the "so-logs-" template group
-    # 2. The template name does not correlate to at least one existing component template
-    # In this situation, the script will treat the skipped template as a temporary failure
-    # and allow the templates to be loaded again on the next run or highstate, whichever 
-    # comes first.
-    COMPONENT_LIST=$(so-elasticsearch-component-templates-list)
-    for i in $pattern; do
-      TEMPLATE=${i::-14}
-      COMPONENT_PATTERN=${TEMPLATE:3}
-      MATCH=$(echo "$TEMPLATE" | grep -E "^so-logs-|^so-metrics" | grep -vE "detections|osquery")
-      if [[ -n "$MATCH" && ! "$COMPONENT_LIST" =~ "$COMPONENT_PATTERN" && ! "$COMPONENT_PATTERN" =~ \.generic|logs-winlog\.winlog ]]; then
-        load_failures=$((load_failures+1))
-        echo "Component template does not exist for $COMPONENT_PATTERN. The index template will not be loaded. Load failures: $load_failures"
-      else
-        load_template "_index_template/$TEMPLATE" "$i"
-      fi
-    done
-  else
-    {% if GLOBALS.role == 'so-heavynode' %}
-    echo "Common template does not exist. Exiting..."
-    {% else %}
-    echo "Elastic Fleet not configured. Exiting..."
-    {% endif %}
-    exit 0
-  fi
-  
-  cd - >/dev/null
-
-  if [[ $load_failures -eq 0 ]]; then
-    echo "All templates loaded successfully"
-    touch $STATE_FILE_SUCCESS
-  else
-    echo "Encountered $load_failures templates that were unable to load, likely due to missing dependencies that will be available later; will retry on next highstate"
-  fi
-else
-  echo "Templates already loaded"
-fi

salt/firewall/containers.map.jinja

@@ -11,6 +11,7 @@
      'so-kratos',
      'so-hydra',
      'so-nginx',
+     'so-postgres',
      'so-redis',
      'so-soc',
      'so-strelka-coordinator',
@@ -34,6 +35,7 @@
      'so-hydra',
      'so-logstash',
      'so-nginx',
+     'so-postgres',
      'so-redis',
      'so-soc',
      'so-strelka-coordinator',
@@ -77,6 +79,7 @@
      'so-kratos',
      'so-hydra',
      'so-nginx',
+     'so-postgres',
      'so-soc'
 ] %}
 

salt/firewall/defaults.yaml

@@ -98,6 +98,10 @@ firewall:
       tcp:
         - 8086
       udp: []
+    postgres:
+      tcp:
+        - 5432
+      udp: []
     kafka_controller:
       tcp:
         - 9093
@@ -193,6 +197,7 @@ firewall:
                 - kibana
                 - redis
                 - influxdb
+                - postgres
                 - elasticsearch_rest
                 - elasticsearch_node
                 - localrules
@@ -379,6 +384,7 @@ firewall:
                 - kibana
                 - redis
                 - influxdb
+                - postgres
                 - elasticsearch_rest
                 - elasticsearch_node
                 - docker_registry
@@ -590,6 +596,7 @@ firewall:
                 - kibana
                 - redis
                 - influxdb
+                - postgres
                 - elasticsearch_rest
                 - elasticsearch_node
                 - docker_registry
@@ -799,6 +806,7 @@ firewall:
                 - kibana
                 - redis
                 - influxdb
+                - postgres
                 - elasticsearch_rest
                 - elasticsearch_node
                 - docker_registry
@@ -1011,6 +1019,7 @@ firewall:
                 - kibana
                 - redis
                 - influxdb
+                - postgres
                 - elasticsearch_rest
                 - elasticsearch_node
                 - docker_registry

salt/firewall/map.jinja

@@ -1,5 +1,6 @@
 {% from 'vars/globals.map.jinja' import GLOBALS %}
 {% from 'docker/docker.map.jinja' import DOCKERMERGED %}
+{% from 'telegraf/map.jinja' import TELEGRAFMERGED %}
 {% import_yaml 'firewall/defaults.yaml' as FIREWALL_DEFAULT %}
 
 {# add our ip to self #}
@@ -55,4 +56,16 @@
 
 {% endif %}
 
+{# Open Postgres (5432) to minion hostgroups when Telegraf is configured to write to Postgres #}
+{% set TG_OUT = TELEGRAFMERGED.output | upper %}
+{% if TG_OUT in ['POSTGRES', 'BOTH'] %}
+{%   if role.startswith('manager') or role == 'standalone' or role == 'eval' %}
+{%     for r in ['sensor', 'searchnode', 'heavynode', 'receiver', 'fleet', 'idh', 'desktop', 'import'] %}
+{%       if FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups[r] is defined %}
+{%         do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups[r].portgroups.append('postgres') %}
+{%       endif %}
+{%     endfor %}
+{%   endif %}
+{% endif %}
+
 {% set FIREWALL_MERGED = salt['pillar.get']('firewall', FIREWALL_DEFAULT.firewall, merge=True) %}

salt/global/soc_global.yaml

@@ -11,18 +11,14 @@ global:
     regexFailureMessage: You must enter a valid IP address or CIDR.
   mdengine:
     description: Which engine to use for meta data generation. Options are ZEEK and SURICATA.
-    regex: ^(ZEEK|SURICATA)$
     options:
       - ZEEK
       - SURICATA
-    regexFailureMessage: You must enter either ZEEK or SURICATA.
     global: True
   pcapengine:
     description: Which engine to use for generating pcap. Currently only SURICATA is supported.
-    regex: ^(SURICATA)$
     options:
       - SURICATA
-    regexFailureMessage: You must enter either SURICATA.
     global: True
   ids:
     description: Which IDS engine to use. Currently only Suricata is supported.
@@ -42,11 +38,9 @@ global:
     advanced: True
   pipeline:
     description: Sets which pipeline technology for events to use. The use of Kafka requires a Security Onion Pro license.
-    regex: ^(REDIS|KAFKA)$
     options:
       - REDIS
       - KAFKA
-    regexFailureMessage: You must enter either REDIS or KAFKA.
     global: True
     advanced: True
   repo_host:

salt/influxdb/soc_influxdb.yaml

@@ -85,7 +85,10 @@ influxdb:
       description: The log level to use for outputting log statements. Allowed values are debug, info, or error.
       global: True
       advanced: false
-      regex: ^(info|debug|error)$
+      options:
+      - info
+      - debug
+      - error
       helpLink: influxdb
     metrics-disabled:
       description: If true, the HTTP endpoint that exposes internal InfluxDB metrics will be inaccessible.
@@ -140,7 +143,9 @@ influxdb:
       description: Determines the type of storage used for secrets. Allowed values are bolt or vault.
       global: True
       advanced: True
-      regex: ^(bolt|vault)$
+      options:
+      - bolt
+      - vault
       helpLink: influxdb
     session-length:
       description: Number of minutes that a user login session can remain authenticated. 
@@ -260,7 +265,9 @@ influxdb:
       description: The type of data store to use for HTTP resources. Allowed values are disk or memory. Memory should not be used for production Security Onion installations.
       global: True
       advanced: True
-      regex: ^(disk|memory)$
+      options:
+      - disk
+      - memory
       helpLink: influxdb
     tls-cert: 
       description: The container path to the certificate to use for TLS encryption of the HTTP requests and responses.

salt/kafka/soc_kafka.yaml

@@ -131,7 +131,10 @@ kafka:
       ssl_x_keystore_x_type:
         description: The key store file format.
         title: ssl.keystore.type
-        regex: ^(JKS|PKCS12|PEM)$
+        options:
+        - JKS
+        - PKCS12
+        - PEM
         helpLink: kafka
       ssl_x_truststore_x_location:
         description: The trust store file location within the Docker container.
@@ -160,7 +163,11 @@ kafka:
       security_x_protocol:
         description: 'Broker communication protocol. Options are: SASL_SSL, PLAINTEXT, SSL, SASL_PLAINTEXT'
         title: security.protocol
-        regex: ^(SASL_SSL|PLAINTEXT|SSL|SASL_PLAINTEXT)
+        options:
+        - SASL_SSL
+        - PLAINTEXT
+        - SSL
+        - SASL_PLAINTEXT
         helpLink: kafka
       ssl_x_keystore_x_location:
         description: The key store file location within the Docker container.
@@ -174,7 +181,10 @@ kafka:
       ssl_x_keystore_x_type:
         description: The key store file format.
         title: ssl.keystore.type
-        regex: ^(JKS|PKCS12|PEM)$
+        options:
+        - JKS
+        - PKCS12
+        - PEM
         helpLink: kafka
       ssl_x_truststore_x_location:
         description: The trust store file location within the Docker container.

salt/kibana/tools/sbin_jinja/so-kibana-space-defaults

@@ -9,5 +9,5 @@ SESSIONCOOKIE=$(curl -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http:
 # Disable certain Features from showing up in the Kibana UI
 echo
 echo "Setting up default Kibana Space:"
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["ml","enterpriseSearch","logs","infrastructure","apm","uptime","monitoring","stackAlerts","actions","securitySolutionCasesV3","inventory","dataQuality","searchSynonyms","enterpriseSearchApplications","enterpriseSearchAnalytics","securitySolutionTimeline","securitySolutionNotes","entityManager"]} ' >> /opt/so/log/kibana/misc.log
+curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["ml","enterpriseSearch","logs","infrastructure","apm","uptime","monitoring","stackAlerts","actions","securitySolutionCasesV3","inventory","dataQuality","searchSynonyms","searchQueryRules","enterpriseSearchApplications","enterpriseSearchAnalytics","securitySolutionTimeline","securitySolutionNotes","securitySolutionRulesV1","entityManager","streams","cloudConnect","slo"]} ' >> /opt/so/log/kibana/misc.log
 echo

salt/kratos/soc_kratos.yaml

@@ -21,8 +21,12 @@ kratos:
         description: "Specify the provider type. Required. Valid values are: auth0, generic, github, google, microsoft"
         global: True
         forcedType: string
-        regex: "auth0|generic|github|google|microsoft"
-        regexFailureMessage: "Valid values are: auth0, generic, github, google, microsoft"
+        options:
+        - auth0
+        - generic
+        - github
+        - google
+        - microsoft
         helpLink: oidc
       client_id: 
         description: Specify the client ID, also referenced as the application ID. Required.
@@ -43,8 +47,9 @@ kratos:
         description: The source of the subject identifier. Typically 'userinfo'. Only used when provider is 'microsoft'.
         global: True
         forcedType: string
-        regex: me|userinfo
-        regexFailureMessage: "Valid values are: me, userinfo"
+        options:
+        - me
+        - userinfo
         helpLink: oidc
       auth_url: 
         description: Provider's auth URL. Required when provider is 'generic'.

salt/manager/tools/sbin/so-minion

@@ -133,7 +133,7 @@ function getinstallinfo() {
 		return 1
 	fi
 
-	export $(echo "$INSTALLVARS" | xargs)
+	while read -r var; do export "$var"; done <<< "$INSTALLVARS"
 	if [ $? -ne 0 ]; then
 		log "ERROR" "Failed to source install variables"
 		return 1
@@ -281,6 +281,39 @@ function deleteMinionFiles () {
 	fi
 }
 
+# Remove this minion's postgres Telegraf credential from the shared creds
+# pillar and drop the matching role in Postgres. Always returns 0 so a dead
+# or unreachable so-postgres doesn't block minion deletion — in that case we
+# log a warning and leave the role behind for manual cleanup.
+function remove_postgres_telegraf_from_minion() {
+	local MINION_SAFE
+	MINION_SAFE=$(echo "$MINION_ID" | tr '.-' '__' | tr '[:upper:]' '[:lower:]')
+	local PG_USER="so_telegraf_${MINION_SAFE}"
+
+	log "INFO" "Removing postgres telegraf cred for $MINION_ID"
+
+	so-telegraf-cred remove "$MINION_ID" >/dev/null 2>&1 || true
+
+	if docker ps --format '{{.Names}}' 2>/dev/null | grep -q '^so-postgres$'; then
+		if ! docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -U postgres -d so_telegraf >/dev/null 2>&1 <<EOSQL
+DO \$\$
+BEGIN
+    IF EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = '$PG_USER') THEN
+        EXECUTE format('REASSIGN OWNED BY %I TO so_telegraf', '$PG_USER');
+        EXECUTE format('DROP OWNED BY %I', '$PG_USER');
+        EXECUTE format('DROP ROLE %I', '$PG_USER');
+    END IF;
+END
+\$\$;
+EOSQL
+		then
+			log "WARN" "Failed to drop postgres role $PG_USER; pillar entry was removed — drop manually if the role persists"
+		fi
+	else
+		log "WARN" "so-postgres container is not running; skipping DB role cleanup for $PG_USER"
+	fi
+}
+
 # Create the minion file
 function ensure_socore_ownership() {
 	log "INFO" "Setting socore ownership on minion files"
@@ -542,6 +575,17 @@ function add_telegraf_to_minion() {
         log "ERROR" "Failed to add telegraf configuration to $PILLARFILE"
         return 1
     fi
+
+    # Provision the per-minion postgres Telegraf credential in the shared
+    # telegraf/creds.sls pillar. so-telegraf-cred is the only writer; it
+    # generates a password on first add and is a no-op on re-add so the cred
+    # is stable across repeated so-minion runs. postgres.telegraf_users on the
+    # manager creates/updates the DB role from the same pillar.
+    so-telegraf-cred add "$MINION_ID"
+    if [ $? -ne 0 ]; then
+        log "ERROR" "Failed to provision postgres telegraf cred for $MINION_ID"
+        return 1
+    fi
 }
 
 function add_influxdb_to_minion() {
@@ -1069,6 +1113,7 @@ case "$OPERATION" in
 
 	"delete")
 		log "INFO" "Removing minion $MINION_ID"
+		remove_postgres_telegraf_from_minion
 		deleteMinionFiles || {
 			log "ERROR" "Failed to delete minion files for $MINION_ID"
 			exit 1

salt/manager/tools/sbin/so-telegraf-cred

@@ -0,0 +1,54 @@
+#!/bin/bash
+
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+# Single writer for the Telegraf Postgres credentials pillar. Thin wrapper
+# around so-yaml.py that generates a password on first add and no-ops on
+# re-add so the cred is stable across repeated so-minion runs.
+#
+# Note: so-yaml.py splits keys on '.' with no escape. SO minion ids are
+# dot-free by construction (setup/so-functions:1884 takes the short_name
+# before the first '.'), so using the raw minion id as the key is safe.
+
+CREDS=/opt/so/saltstack/local/pillar/telegraf/creds.sls
+
+usage() {
+    echo "Usage: $0 <add|remove> <minion_id>" >&2
+    exit 2
+}
+
+seed_creds_file() {
+    mkdir -p "$(dirname "$CREDS")" || return 1
+    if [[ ! -f "$CREDS" ]]; then
+        (umask 027 && printf 'telegraf:\n  postgres_creds: {}\n' > "$CREDS") || return 1
+        chown socore:socore "$CREDS" 2>/dev/null || true
+        chmod 640 "$CREDS" || return 1
+    fi
+}
+
+OP=$1
+MID=$2
+[[ -z "$OP" || -z "$MID" ]] && usage
+
+case "$OP" in
+    add)
+        SAFE=$(echo "$MID" | tr '.-' '__' | tr '[:upper:]' '[:lower:]')
+        seed_creds_file || exit 1
+        if so-yaml.py get -r "$CREDS" "telegraf.postgres_creds.${MID}.user" >/dev/null 2>&1; then
+            exit 0
+        fi
+        PASS=$(tr -dc 'A-Za-z0-9~!@#^&*()_=+[]|;:,.<>?-' < /dev/urandom | head -c 72)
+        so-yaml.py replace "$CREDS" "telegraf.postgres_creds.${MID}.user" "so_telegraf_${SAFE}" >/dev/null
+        so-yaml.py replace "$CREDS" "telegraf.postgres_creds.${MID}.pass" "$PASS" >/dev/null
+        ;;
+    remove)
+        [[ -f "$CREDS" ]] || exit 0
+        so-yaml.py remove "$CREDS" "telegraf.postgres_creds.${MID}" >/dev/null 2>&1 || true
+        ;;
+    *)
+        usage
+        ;;
+esac

salt/manager/tools/sbin/so-yaml.py

@@ -39,9 +39,16 @@ def showUsage(args):
 
 
 def loadYaml(filename):
-    file = open(filename, "r")
+    try:
+        with open(filename, "r") as file:
             content = file.read()
             return yaml.safe_load(content)
+    except FileNotFoundError:
+        print(f"File not found: {filename}", file=sys.stderr)
+        sys.exit(1)
+    except Exception as e:
+        print(f"Error reading file {filename}: {e}", file=sys.stderr)
+        sys.exit(1)
 
 
 def writeYaml(filename, content):
@@ -285,6 +292,7 @@ def add(args):
 def removeKey(content, key):
     pieces = key.split(".", 1)
     if len(pieces) > 1:
+        if pieces[0] in content:
             removeKey(content[pieces[0]], pieces[1])
     else:
         content.pop(key, None)

salt/manager/tools/sbin/so-yaml_test.py

@@ -973,3 +973,21 @@ class TestReplaceListObject(unittest.TestCase):
 
         expected = "key1:\n- id: '1'\n  status: updated\n- id: '2'\n  status: inactive\n"
         self.assertEqual(actual, expected)
+
+
+class TestLoadYaml(unittest.TestCase):
+
+    def test_load_yaml_missing_file(self):
+        with patch('sys.exit', new=MagicMock()) as sysmock:
+            with patch('sys.stderr', new=StringIO()) as mock_stderr:
+                soyaml.loadYaml("/tmp/so-yaml_test-does-not-exist.yaml")
+                sysmock.assert_called_with(1)
+                self.assertIn("File not found:", mock_stderr.getvalue())
+
+    def test_load_yaml_read_error(self):
+        with patch('sys.exit', new=MagicMock()) as sysmock:
+            with patch('sys.stderr', new=StringIO()) as mock_stderr:
+                with patch('builtins.open', side_effect=PermissionError("denied")):
+                    soyaml.loadYaml("/tmp/so-yaml_test-unreadable.yaml")
+                    sysmock.assert_called_with(1)
+                    self.assertIn("Error reading file", mock_stderr.getvalue())

salt/manager/tools/sbin/soup

@@ -305,7 +305,7 @@ clone_to_tmp() {
   # Make a temp location for the files
   mkdir -p /tmp/sogh
   cd /tmp/sogh
-  SOUP_BRANCH="-b 2.4/main"
+  SOUP_BRANCH="-b 3/main"
   if [ -n "$BRANCH" ]; then
     SOUP_BRANCH="-b $BRANCH"
   fi
@@ -363,6 +363,7 @@ preupgrade_changes() {
     echo "Checking to see if changes are needed."
 
     [[ "$INSTALLEDVERSION" =~ ^2\.4\.21[0-9]+$ ]] && up_to_3.0.0   
+    [[ "$INSTALLEDVERSION" == "3.0.0" ]] && up_to_3.1.0
     true
 }
 
@@ -371,6 +372,7 @@ postupgrade_changes() {
     echo "Running post upgrade processes."
 
     [[ "$POSTVERSION" =~ ^2\.4\.21[0-9]+$ ]] && post_to_3.0.0
+    [[ "$POSTVERSION" == "3.0.0" ]] && post_to_3.1.0
     true
 }
 
@@ -445,7 +447,6 @@ migrate_pcap_to_suricata() {
 }
 
 up_to_3.0.0() {
-  determine_elastic_agent_upgrade
   migrate_pcap_to_suricata
 
   INSTALLEDVERSION=3.0.0
@@ -469,6 +470,83 @@ post_to_3.0.0() {
 
 ### 3.0.0 End ###
 
+### 3.1.0 Scripts ###
+
+elasticsearch_backup_index_templates() {
+  echo "Backing up current elasticsearch index templates in /opt/so/conf/elasticsearch/templates/index/ to /nsm/backup/3.0.0_elasticsearch_index_templates.tar.gz"
+  tar -czf /nsm/backup/3.0.0_elasticsearch_index_templates.tar.gz -C /opt/so/conf/elasticsearch/templates/index/ .
+}
+
+ensure_postgres_local_pillar() {
+  # Postgres was added as a service after 3.0.0, so the new pillar/top.sls
+  # references postgres.soc_postgres / postgres.adv_postgres unconditionally.
+  # Managers upgrading from 3.0.0 have no /opt/so/saltstack/local/pillar/postgres/
+  # (make_some_dirs only runs at install time), so the stubs must be created
+  # here before salt-master restarts against the new top.sls.
+  echo "Ensuring postgres local pillar stubs exist."
+  local dir=/opt/so/saltstack/local/pillar/postgres
+  mkdir -p "$dir"
+  [[ -f "$dir/soc_postgres.sls" ]] || touch "$dir/soc_postgres.sls"
+  [[ -f "$dir/adv_postgres.sls" ]] || touch "$dir/adv_postgres.sls"
+  chown -R socore:socore "$dir"
+}
+
+ensure_postgres_secret() {
+  # On a fresh install, generate_passwords + secrets_pillar seed
+  # secrets:postgres_pass in /opt/so/saltstack/local/pillar/secrets.sls. That
+  # code path is skipped on upgrade (secrets.sls already exists from 3.0.0
+  # with import_pass/influx_pass but no postgres_pass), so the postgres
+  # container's POSTGRES_PASSWORD_FILE and SOC's PG_ADMIN_PASS would be empty
+  # after highstate. Generate one now if missing.
+  local secrets_file=/opt/so/saltstack/local/pillar/secrets.sls
+  if [[ ! -f "$secrets_file" ]]; then
+    echo "WARNING: $secrets_file missing; skipping postgres_pass backfill."
+    return 0
+  fi
+  if so-yaml.py get -r "$secrets_file" secrets.postgres_pass >/dev/null 2>&1; then
+    echo "secrets.postgres_pass already set; leaving as-is."
+    return 0
+  fi
+  echo "Seeding secrets.postgres_pass in $secrets_file."
+  so-yaml.py add "$secrets_file" secrets.postgres_pass "$(get_random_value)"
+  chown socore:socore "$secrets_file"
+}
+
+up_to_3.1.0() {
+  ensure_postgres_local_pillar
+  ensure_postgres_secret
+  determine_elastic_agent_upgrade
+  elasticsearch_backup_index_templates
+  # Clear existing component template state file.
+  rm -f /opt/so/state/esfleet_component_templates.json
+
+
+  INSTALLEDVERSION=3.1.0
+}
+
+post_to_3.1.0() {
+  /usr/sbin/so-kibana-space-defaults
+
+  # Backfill the Telegraf creds pillar for every accepted minion. so-telegraf-cred
+  # add is idempotent — it no-ops when an entry already exists — so this is safe
+  # to run on every soup. The subsequent state.apply creates/updates the matching
+  # Postgres roles from the reconciled pillar.
+  echo "Reconciling Telegraf Postgres creds for accepted minions."
+  for mid in $(salt-key --out=json --list=accepted 2>/dev/null | jq -r '.minions[]?' 2>/dev/null); do
+    [[ -n "$mid" ]] || continue
+    /usr/sbin/so-telegraf-cred add "$mid" || echo "  warning: so-telegraf-cred add $mid failed" >&2
+  done
+  # Run through the master (not --local) so state compilation uses the
+  # master's configured file_roots; the manager's /etc/salt/minion has no
+  # file_roots of its own and --local would fail with "No matching sls found".
+  salt-call state.apply postgres.telegraf_users queue=True || true
+
+  POSTVERSION=3.1.0
+}
+
+### 3.1.0 End ###
+
+
 repo_sync() {
   echo "Sync the local repo."
   su socore -c '/usr/sbin/so-repo-sync' || fail "Unable to complete so-repo-sync."
@@ -728,12 +806,12 @@ verify_es_version_compatibility() {
     local is_active_intermediate_upgrade=1
     # supported upgrade paths for SO-ES versions
     declare -A es_upgrade_map=(
-	    ["8.18.8"]="9.0.8"
+        ["9.0.8"]="9.3.3"
     )
 
     # Elasticsearch MUST upgrade through these versions
     declare -A es_to_so_version=(
-	    ["8.18.8"]="2.4.190-20251024"
+        ["9.0.8"]="3.0.0-20260331"
     )
 
     # Get current Elasticsearch version
@@ -745,26 +823,17 @@ verify_es_version_compatibility() {
         exit 160
     fi
 
-    if ! target_es_version_raw=$(so-yaml.py get $UPDATE_DIR/salt/elasticsearch/defaults.yaml elasticsearch.version); then
-        # so-yaml.py failed to get the ES version from upgrade versions elasticsearch/defaults.yaml file. Likely they are upgrading to an SO version older than 2.4.110 prior to the ES version pinning and should be OKAY to continue with the upgrade.
-
-        # if so-yaml.py failed to get the ES version AND the version we are upgrading to is newer than 2.4.110 then we should bail
-        if [[ $(cat $UPDATE_DIR/VERSION | cut -d'.' -f3) > 110 ]]; then
+    if ! target_es_version=$(so-yaml.py get -r $UPDATE_DIR/salt/elasticsearch/defaults.yaml elasticsearch.version); then
         echo "Couldn't determine the target Elasticsearch version (post soup version) to ensure compatibility with current Elasticsearch version. Exiting"
 
         exit 160
     fi
 
-        # allow upgrade to version < 2.4.110 without checking ES version compatibility
-        return 0
-    else
-        target_es_version=$(sed -n '1p' <<< "$target_es_version_raw")
-    fi
-
     for statefile in "${es_required_version_statefile_base}"-*; do
         [[ -f $statefile ]] || continue
 
-        local es_required_version_statefile_value=$(cat "$statefile")
+        local es_required_version_statefile_value
+        es_required_version_statefile_value=$(cat "$statefile")
 
         if [[ "$es_required_version_statefile_value" == "$target_es_version" ]]; then
             echo "Intermediate upgrade to ES $target_es_version is in progress. Skipping Elasticsearch version compatibility check."
@@ -773,7 +842,7 @@ verify_es_version_compatibility() {
         fi
 
         # use sort to check if es_required_statefile_value is < the current es_version.
-        if [[ "$(printf '%s\n' $es_required_version_statefile_value $es_version | sort -V | head -n1)" == "$es_required_version_statefile_value" ]]; then
+        if [[ "$(printf '%s\n' "$es_required_version_statefile_value" "$es_version" | sort -V | head -n1)" == "$es_required_version_statefile_value" ]]; then
             rm -f "$statefile"
             continue
         fi
@@ -784,8 +853,7 @@ verify_es_version_compatibility() {
 
         echo -e "\n##############################################################################################################################\n"
         echo "A previously required intermediate Elasticsearch upgrade was detected. Verifying that all Searchnodes/Heavynodes have successfully upgraded Elasticsearch to $es_required_version_statefile_value before proceeding with soup to avoid potential data loss! This command can take up to an hour to complete."
-        timeout --foreground 4000 bash "$es_verification_script" "$es_required_version_statefile_value" "$statefile"
-        if [[ $? -ne 0 ]]; then
+        if ! timeout --foreground 4000 bash "$es_verification_script" "$es_required_version_statefile_value" "$statefile"; then
             echo -e "\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
 
             echo "A previous required intermediate Elasticsearch upgrade to $es_required_version_statefile_value has yet to successfully complete across the grid. Please allow time for all Searchnodes/Heavynodes to have upgraded Elasticsearch to $es_required_version_statefile_value before running soup again to avoid potential data loss!"
@@ -802,6 +870,7 @@ verify_es_version_compatibility() {
         return 0
     fi
 
+    # shellcheck disable=SC2076 # Do not want a regex here eg usage " 8.18.8 9.0.8 " =~ " 9.0.8 "
     if [[ " ${es_upgrade_map[$es_version]} " =~ " $target_es_version " || "$es_version" == "$target_es_version" ]]; then
         # supported upgrade
         return 0
@@ -810,7 +879,7 @@ verify_es_version_compatibility() {
         if [[ -z "$compatible_versions" ]]; then
             # If current ES version is not explicitly defined in the upgrade map, we know they have an intermediate upgrade to do.
             # We default to the lowest ES version defined in es_to_so_version as $first_es_required_version
-            local first_es_required_version=$(printf '%s\n' "${!es_to_so_version[@]}" | sort -V | head -n1)
+            first_es_required_version=$(printf '%s\n' "${!es_to_so_version[@]}" | sort -V | head -n1)
             next_step_so_version=${es_to_so_version[$first_es_required_version]}
             required_es_upgrade_version="$first_es_required_version"
         else
@@ -829,7 +898,7 @@ verify_es_version_compatibility() {
         if [[ $is_airgap -eq 0 ]]; then
             run_airgap_intermediate_upgrade
         else
-            if [[ ! -z $ISOLOC ]]; then
+            if [[ -n $ISOLOC ]]; then
                 originally_requested_iso_location="$ISOLOC"
             fi
             # Make sure ISOLOC is not set. Network installs that used soup -f would have ISOLOC set.
@@ -861,7 +930,8 @@ wait_for_salt_minion_with_restart() {
 }
 
 run_airgap_intermediate_upgrade() {
-    local originally_requested_so_version=$(cat $UPDATE_DIR/VERSION)
+    local originally_requested_so_version
+    originally_requested_so_version=$(cat "$UPDATE_DIR/VERSION")
     # preserve ISOLOC value, so we can try to use it post intermediate upgrade
     local originally_requested_iso_location="$ISOLOC"
 
@@ -873,7 +943,8 @@ run_airgap_intermediate_upgrade() {
 
     while [[ -z "$next_iso_location" ]] || [[ ! -f "$next_iso_location" && ! -b "$next_iso_location" ]]; do
         # List removable devices if any are present
-        local removable_devices=$(lsblk -no PATH,SIZE,TYPE,MOUNTPOINTS,RM | awk '$NF==1')
+        local removable_devices
+        removable_devices=$(lsblk -no PATH,SIZE,TYPE,MOUNTPOINTS,RM | awk '$NF==1')
         if [[ -n "$removable_devices" ]]; then
             echo "PATH        SIZE    TYPE    MOUNTPOINTS    RM"
             echo "$removable_devices"
@@ -894,21 +965,21 @@ run_airgap_intermediate_upgrade() {
 
     echo "Using $next_iso_location for required intermediary upgrade."
     exec bash <<EOF
-        ISOLOC=$next_iso_location soup -y && \
-        ISOLOC=$next_iso_location soup -y && \
+        ISOLOC="$next_iso_location" soup -y && \
+        ISOLOC="$next_iso_location" soup -y && \
 
         echo -e "\n##############################################################################################################################\n" && \
         echo -e "Verifying Elasticsearch was successfully upgraded to $required_es_upgrade_version across the grid. This part can take a while as Searchnodes/Heavynodes sync up with the Manager! \n\nOnce verification completes the next soup will begin automatically. If verification takes longer than 1 hour it will stop waiting and your grid will remain at $next_step_so_version. Allowing for all Searchnodes/Heavynodes to upgrade Elasticsearch to the required version on their own time.\n" && \
 
-        timeout --foreground 4000 bash /tmp/so_intermediate_upgrade_verification.sh $required_es_upgrade_version $es_required_version_statefile && \
+        timeout --foreground 4000 bash /tmp/so_intermediate_upgrade_verification.sh "$required_es_upgrade_version" "$es_required_version_statefile" && \
 
         echo -e "\n##############################################################################################################################\n" && \
 
         # automatically start the next soup if the original ISO isn't using the same block device we just used
         if [[ -n "$originally_requested_iso_location" ]] && [[ "$originally_requested_iso_location" != "$next_iso_location" ]]; then
             umount /tmp/soagupdate
-            ISOLOC=$originally_requested_iso_location soup -y && \
-            ISOLOC=$originally_requested_iso_location soup -y
+            ISOLOC="$originally_requested_iso_location" soup -y && \
+            ISOLOC="$originally_requested_iso_location" soup -y
         else
             echo "Could not automatically start next soup to $originally_requested_so_version. Soup will now exit here at $(cat /etc/soversion)" && \
 
@@ -924,29 +995,29 @@ run_network_intermediate_upgrade() {
     if [[ -n "$BRANCH" ]]; then
         local originally_requested_so_branch="$BRANCH"
     else
-        local originally_requested_so_branch="2.4/main"
+        local originally_requested_so_branch="3/main"
     fi
 
     echo "Starting automated intermediate upgrade to $next_step_so_version."
     echo "After completion, the system will automatically attempt to upgrade to the latest version."
     echo -e "\n##############################################################################################################################\n"
     exec bash << EOF
-        BRANCH=$next_step_so_version soup -y && \
-        BRANCH=$next_step_so_version soup -y && \
+        BRANCH="$next_step_so_version" soup -y && \
+        BRANCH="$next_step_so_version" soup -y && \
 
         echo -e "\n##############################################################################################################################\n" && \
         echo -e "Verifying Elasticsearch was successfully upgraded to $required_es_upgrade_version across the grid. This part can take a while as Searchnodes/Heavynodes sync up with the Manager! \n\nOnce verification completes the next soup will begin automatically. If verification takes longer than 1 hour it will stop waiting and your grid will remain at $next_step_so_version. Allowing for all Searchnodes/Heavynodes to upgrade Elasticsearch to the required version on their own time.\n" && \
 
-        timeout --foreground 4000 bash /tmp/so_intermediate_upgrade_verification.sh $required_es_upgrade_version $es_required_version_statefile && \
+        timeout --foreground 4000 bash /tmp/so_intermediate_upgrade_verification.sh "$required_es_upgrade_version" "$es_required_version_statefile" && \
 
         echo -e "\n##############################################################################################################################\n" && \
         if [[ -n "$originally_requested_iso_location" ]]; then
             # nonairgap soup that used -f originally, runs intermediate upgrade using network + BRANCH, later coming back to the original ISO for the last soup
-            ISOLOC=$originally_requested_iso_location soup -y && \
-            ISOLOC=$originally_requested_iso_location soup -y
+            ISOLOC="$originally_requested_iso_location" soup -y && \
+            ISOLOC="$originally_requested_iso_location" soup -y
         else
-            BRANCH=$originally_requested_so_branch soup -y && \
-            BRANCH=$originally_requested_so_branch soup -y
+            BRANCH="$originally_requested_so_branch" soup -y && \
+            BRANCH="$originally_requested_so_branch" soup -y
         fi
         echo -e "\n##############################################################################################################################\n"
 EOF

salt/orch/deploy_newnode.sls

@@ -25,8 +25,33 @@ manager_run_es_soc:
         - salt: {{NEWNODE}}_update_mine
 {% endif %}
 
+# so-minion has already added the new minion's entry to telegraf/creds.sls
+# via so-telegraf-cred before this orch fires. Reconcile the Postgres role
+# on the manager so the new minion can authenticate on its first highstate,
+# then refresh the minion's pillar so its telegraf.conf renders with the
+# freshly-written cred.
+manager_create_postgres_telegraf_role:
+  salt.state:
+    - tgt: {{ MANAGER }}
+    - sls:
+      - postgres.telegraf_users
+    - queue: True
+    - require:
+      - salt: {{NEWNODE}}_update_mine
+
+{{NEWNODE}}_refresh_pillar:
+  salt.function:
+    - name: saltutil.refresh_pillar
+    - tgt: {{ NEWNODE }}
+    - kwarg:
+        wait: True
+    - require:
+      - salt: manager_create_postgres_telegraf_role
+
 {{NEWNODE}}_run_highstate:
   salt.state:
     - tgt: {{ NEWNODE }}
     - highstate: True
     - queue: True
+    - require:
+      - salt: {{NEWNODE}}_refresh_pillar

salt/postgres/auth.sls

@@ -0,0 +1,37 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{% from 'allowed_states.map.jinja' import allowed_states %}
+{% if sls in allowed_states %}
+
+  {% set DIGITS = "1234567890" %}
+  {% set LOWERCASE = "qwertyuiopasdfghjklzxcvbnm" %}
+  {% set UPPERCASE = "QWERTYUIOPASDFGHJKLZXCVBNM" %}
+  {% set SYMBOLS = "~!@#^&*()-_=+[]|;:,.<>?" %}
+  {% set CHARS = DIGITS~LOWERCASE~UPPERCASE~SYMBOLS %}
+  {% set so_postgres_user_pass = salt['pillar.get']('postgres:auth:users:so_postgres_user:pass', salt['random.get_str'](72, chars=CHARS)) %}
+
+# Admin cred only. Per-minion Telegraf creds live in telegraf/creds.sls,
+# managed by /usr/sbin/so-telegraf-cred (called from so-minion).
+postgres_auth_pillar:
+  file.managed:
+    - name: /opt/so/saltstack/local/pillar/postgres/auth.sls
+    - mode: 640
+    - reload_pillar: True
+    - contents: |
+        postgres:
+          auth:
+            users:
+              so_postgres_user:
+                user: so_postgres
+                pass: "{{ so_postgres_user_pass }}"
+    - show_changes: False
+{% else %}
+
+{{sls}}_state_not_allowed:
+  test.fail_without_changes:
+    - name: {{sls}}_state_not_allowed
+
+{% endif %}

salt/postgres/config.sls

@@ -0,0 +1,111 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{% from 'allowed_states.map.jinja' import allowed_states %}
+{% if sls.split('.')[0] in allowed_states %}
+{%   from 'postgres/map.jinja' import PGMERGED %}
+
+# Postgres Setup
+postgresconfdir:
+  file.directory:
+    - name: /opt/so/conf/postgres
+    - user: 939
+    - group: 939
+    - makedirs: True
+
+postgressecretsdir:
+  file.directory:
+    - name: /opt/so/conf/postgres/secrets
+    - user: 939
+    - group: 939
+    - mode: 700
+    - require:
+      - file: postgresconfdir
+
+postgresdatadir:
+  file.directory:
+    - name: /nsm/postgres
+    - user: 939
+    - group: 939
+    - makedirs: True
+
+postgreslogdir:
+  file.directory:
+    - name: /opt/so/log/postgres
+    - user: 939
+    - group: 939
+    - makedirs: True
+
+postgresinitdir:
+  file.directory:
+    - name: /opt/so/conf/postgres/init
+    - user: 939
+    - group: 939
+    - require:
+      - file: postgresconfdir
+
+postgresinitusers:
+  file.managed:
+    - name: /opt/so/conf/postgres/init/init-users.sh
+    - source: salt://postgres/files/init-users.sh
+    - user: 939
+    - group: 939
+    - mode: 755
+
+postgresconf:
+  file.managed:
+    - name: /opt/so/conf/postgres/postgresql.conf
+    - source: salt://postgres/files/postgresql.conf.jinja
+    - user: 939
+    - group: 939
+    - template: jinja
+    - defaults:
+        PGMERGED: {{ PGMERGED }}
+
+postgreshba:
+  file.managed:
+    - name: /opt/so/conf/postgres/pg_hba.conf
+    - source: salt://postgres/files/pg_hba.conf
+    - user: 939
+    - group: 939
+    - mode: 640
+
+postgres_super_secret:
+  file.managed:
+    - name: /opt/so/conf/postgres/secrets/postgres_password
+    - user: 939
+    - group: 939
+    - mode: 600
+    - contents_pillar: 'secrets:postgres_pass'
+    - show_changes: False
+    - require:
+      - file: postgressecretsdir
+
+postgres_app_secret:
+  file.managed:
+    - name: /opt/so/conf/postgres/secrets/so_postgres_pass
+    - user: 939
+    - group: 939
+    - mode: 600
+    - contents_pillar: 'postgres:auth:users:so_postgres_user:pass'
+    - show_changes: False
+    - require:
+      - file: postgressecretsdir
+
+postgres_sbin:
+  file.recurse:
+    - name: /usr/sbin
+    - source: salt://postgres/tools/sbin
+    - user: root
+    - group: root
+    - file_mode: 755
+
+{% else %}
+
+{{sls}}_state_not_allowed:
+  test.fail_without_changes:
+    - name: {{sls}}_state_not_allowed
+
+{% endif %}

salt/postgres/defaults.yaml

@@ -0,0 +1,19 @@
+postgres:
+  enabled: True
+  telegraf:
+    retention_days: 14
+  config:
+    listen_addresses: '*'
+    port: 5432
+    max_connections: 100
+    shared_buffers: 256MB
+    ssl: 'on'
+    ssl_cert_file: '/conf/postgres.crt'
+    ssl_key_file: '/conf/postgres.key'
+    ssl_ca_file: '/conf/ca.crt'
+    hba_file: '/conf/pg_hba.conf'
+    log_destination: 'stderr'
+    logging_collector: 'off'
+    log_min_messages: 'warning'
+    shared_preload_libraries: pg_cron
+    cron.database_name: so_telegraf

salt/postgres/disabled.sls

@@ -0,0 +1,33 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{% from 'allowed_states.map.jinja' import allowed_states %}
+{% if sls.split('.')[0] in allowed_states %}
+
+include:
+  - postgres.sostatus
+
+so-postgres:
+  docker_container.absent:
+    - force: True
+
+so-postgres_so-status.disabled:
+  file.comment:
+    - name: /opt/so/conf/so-status/so-status.conf
+    - regex: ^so-postgres$
+
+so_postgres_backup:
+  cron.absent:
+    - name: /usr/sbin/so-postgres-backup > /dev/null 2>&1
+    - identifier: so_postgres_backup
+    - user: root
+
+{% else %}
+
+{{sls}}_state_not_allowed:
+  test.fail_without_changes:
+    - name: {{sls}}_state_not_allowed
+
+{% endif %}

salt/postgres/enabled.sls

@@ -0,0 +1,109 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{% from 'allowed_states.map.jinja' import allowed_states %}
+{% if sls.split('.')[0] in allowed_states %}
+{%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
+{%   set SO_POSTGRES_USER = salt['pillar.get']('postgres:auth:users:so_postgres_user:user', 'so_postgres') %}
+
+include:
+  - postgres.auth
+  - postgres.ssl
+  - postgres.config
+  - postgres.sostatus
+  - postgres.telegraf_users
+
+so-postgres:
+  docker_container.running:
+    - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-postgres:{{ GLOBALS.so_version }}
+    - hostname: so-postgres
+    - networks:
+      - sobridge:
+        - ipv4_address: {{ DOCKERMERGED.containers['so-postgres'].ip }}
+    - port_bindings:
+      {% for BINDING in DOCKERMERGED.containers['so-postgres'].port_bindings %}
+      - {{ BINDING }}
+      {% endfor %}
+    - environment:
+      - POSTGRES_DB=securityonion
+      # Passwords are delivered via mounted 0600 secret files, not plaintext env vars.
+      # The upstream postgres image resolves POSTGRES_PASSWORD_FILE; entrypoint.sh and
+      # init-users.sh resolve SO_POSTGRES_PASS_FILE the same way.
+      - POSTGRES_PASSWORD_FILE=/run/secrets/postgres_password
+      - SO_POSTGRES_USER={{ SO_POSTGRES_USER }}
+      - SO_POSTGRES_PASS_FILE=/run/secrets/so_postgres_pass
+      {% if DOCKERMERGED.containers['so-postgres'].extra_env %}
+        {% for XTRAENV in DOCKERMERGED.containers['so-postgres'].extra_env %}
+      - {{ XTRAENV }}
+        {% endfor %}
+      {% endif %}
+    - binds:
+      - /opt/so/log/postgres/:/log:rw
+      - /nsm/postgres:/var/lib/postgresql/data:rw
+      - /opt/so/conf/postgres/postgresql.conf:/conf/postgresql.conf:ro
+      - /opt/so/conf/postgres/pg_hba.conf:/conf/pg_hba.conf:ro
+      - /opt/so/conf/postgres/secrets:/run/secrets:ro
+      - /opt/so/conf/postgres/init/init-users.sh:/docker-entrypoint-initdb.d/init-users.sh:ro
+      - /etc/pki/postgres.crt:/conf/postgres.crt:ro
+      - /etc/pki/postgres.key:/conf/postgres.key:ro
+      - /etc/pki/tls/certs/intca.crt:/conf/ca.crt:ro
+      {% if DOCKERMERGED.containers['so-postgres'].custom_bind_mounts %}
+        {% for BIND in DOCKERMERGED.containers['so-postgres'].custom_bind_mounts %}
+      - {{ BIND }}
+        {% endfor %}
+      {% endif %}
+    {% if DOCKERMERGED.containers['so-postgres'].extra_hosts %}
+    - extra_hosts:
+      {% for XTRAHOST in DOCKERMERGED.containers['so-postgres'].extra_hosts %}
+      - {{ XTRAHOST }}
+      {% endfor %}
+    {% endif %}
+    {% if DOCKERMERGED.containers['so-postgres'].ulimits %}
+    - ulimits:
+    {%   for ULIMIT in DOCKERMERGED.containers['so-postgres'].ulimits %}
+      - {{ ULIMIT.name }}={{ ULIMIT.soft }}:{{ ULIMIT.hard }}
+    {%   endfor %}
+    {% endif %}
+    - watch:
+      - file: postgresconf
+      - file: postgreshba
+      - file: postgresinitusers
+      - file: postgres_super_secret
+      - file: postgres_app_secret
+      - x509: postgres_crt
+      - x509: postgres_key
+    - require:
+      - file: postgresconf
+      - file: postgreshba
+      - file: postgresinitusers
+      - file: postgres_super_secret
+      - file: postgres_app_secret
+      - x509: postgres_crt
+      - x509: postgres_key
+
+delete_so-postgres_so-status.disabled:
+  file.uncomment:
+    - name: /opt/so/conf/so-status/so-status.conf
+    - regex: ^so-postgres$
+
+so_postgres_backup:
+  cron.present:
+    - name: /usr/sbin/so-postgres-backup > /dev/null 2>&1
+    - identifier: so_postgres_backup
+    - user: root
+    - minute: '5'
+    - hour: '0'
+    - daymonth: '*'
+    - month: '*'
+    - dayweek: '*'
+
+{% else %}
+
+{{sls}}_state_not_allowed:
+  test.fail_without_changes:
+    - name: {{sls}}_state_not_allowed
+
+{% endif %}

salt/postgres/files/init-users.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+set -e
+
+# Create or update application user for SOC platform access
+# This script runs on first database initialization via docker-entrypoint-initdb.d
+# The password is properly escaped to handle special characters
+if [ -z "${SO_POSTGRES_PASS:-}" ] && [ -n "${SO_POSTGRES_PASS_FILE:-}" ] && [ -r "$SO_POSTGRES_PASS_FILE" ]; then
+    SO_POSTGRES_PASS="$(< "$SO_POSTGRES_PASS_FILE")"
+fi
+psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL
+    DO \$\$
+    BEGIN
+        IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = '${SO_POSTGRES_USER}') THEN
+            EXECUTE format('CREATE ROLE %I WITH LOGIN PASSWORD %L', '${SO_POSTGRES_USER}', '${SO_POSTGRES_PASS}');
+        ELSE
+            EXECUTE format('ALTER ROLE %I WITH PASSWORD %L', '${SO_POSTGRES_USER}', '${SO_POSTGRES_PASS}');
+        END IF;
+    END
+    \$\$;
+    GRANT ALL PRIVILEGES ON DATABASE "$POSTGRES_DB" TO "$SO_POSTGRES_USER";
+    -- Lock the SOC database down at the connect layer; PUBLIC gets CONNECT
+    -- by default, which would let per-minion telegraf roles open sessions
+    -- here. They have no schema/table grants inside so reads fail, but
+    -- revoking CONNECT closes the soft edge entirely.
+    REVOKE CONNECT ON DATABASE "$POSTGRES_DB" FROM PUBLIC;
+    GRANT CONNECT ON DATABASE "$POSTGRES_DB" TO "$SO_POSTGRES_USER";
+EOSQL
+
+# Bootstrap the Telegraf metrics database. Per-minion roles + schemas are
+# reconciled on every state.apply by postgres/telegraf_users.sls; this block
+# only ensures the shared database exists on first initialization.
+if ! psql -U "$POSTGRES_USER" -tAc "SELECT 1 FROM pg_database WHERE datname='so_telegraf'" | grep -q 1; then
+    psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -c "CREATE DATABASE so_telegraf"
+fi

salt/postgres/files/pg_hba.conf

@@ -0,0 +1,16 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+#
+# Managed by Salt — do not edit by hand.
+# Client authentication config: only local (Unix socket) connections and TLS-wrapped TCP
+# connections are accepted. Plain-text `host ...` lines are intentionally omitted so a
+# misconfigured client with sslmode=disable cannot negotiate a cleartext session.
+
+# Local connections (Unix socket, container-internal) use peer/trust.
+local   all             all                                     trust
+
+# TCP connections MUST use TLS (hostssl) and authenticate with SCRAM.
+hostssl all             all             0.0.0.0/0               scram-sha-256
+hostssl all             all             ::/0                    scram-sha-256

salt/postgres/files/postgresql.conf.jinja

@@ -0,0 +1,8 @@
+{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+   or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+   https://securityonion.net/license; you may not use this file except in compliance with the
+   Elastic License 2.0. #}
+
+{% for key, value in PGMERGED.config.items() %}
+{{ key }} = '{{ value | string | replace("'", "''") }}'
+{% endfor %}

salt/postgres/init.sls

@@ -0,0 +1,13 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{% from 'postgres/map.jinja' import PGMERGED %}
+
+include:
+{% if PGMERGED.enabled %}
+  - postgres.enabled
+{% else %}
+  - postgres.disabled
+{% endif %}

salt/postgres/map.jinja

@@ -0,0 +1,7 @@
+{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+   or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+   https://securityonion.net/license; you may not use this file except in compliance with the
+   Elastic License 2.0. #}
+
+{% import_yaml 'postgres/defaults.yaml' as PGDEFAULTS %}
+{% set PGMERGED = salt['pillar.get']('postgres', PGDEFAULTS.postgres, merge=True) %}

salt/postgres/soc_postgres.yaml

@@ -0,0 +1,89 @@
+postgres:
+  enabled:
+    description: Whether the PostgreSQL database container is enabled on this grid. Backs the assistant store and the Telegraf metrics database.
+    forcedType: bool
+    readonly: True
+    helpLink: influxdb
+  telegraf:
+    retention_days:
+      description: Number of days of Telegraf metrics to keep in the so_telegraf database. Older partitions are dropped hourly by pg_partman.
+      forcedType: int
+      helpLink: postgres
+  config:
+    max_connections:
+      description: Maximum number of concurrent PostgreSQL connections.
+      forcedType: int
+      global: True
+      helpLink: postgres
+    shared_buffers:
+      description: Amount of memory PostgreSQL uses for shared buffers (e.g. 256MB, 1GB). Raising this improves read cache hit rate at the cost of system RAM.
+      global: True
+      helpLink: postgres
+    log_min_messages:
+      description: Minimum severity of server messages written to the PostgreSQL log.
+      options:
+        - debug1
+        - info
+        - notice
+        - warning
+        - error
+        - log
+        - fatal
+      global: True
+      helpLink: postgres
+    listen_addresses:
+      description: Interfaces PostgreSQL listens on. Must remain '*' so clients on the docker bridge network can connect.
+      global: True
+      advanced: True
+      helpLink: postgres
+    port:
+      description: TCP port PostgreSQL listens on inside the container. Firewall rules and container port mapping assume 5432.
+      forcedType: int
+      global: True
+      advanced: True
+      helpLink: postgres
+    ssl:
+      description: Whether PostgreSQL accepts TLS connections. Must remain 'on' — pg_hba.conf requires hostssl for TCP.
+      global: True
+      advanced: True
+      helpLink: postgres
+    ssl_cert_file:
+      description: Path (inside the container) to the TLS server certificate. Salt-managed.
+      global: True
+      advanced: True
+      helpLink: postgres
+    ssl_key_file:
+      description: Path (inside the container) to the TLS server private key. Salt-managed.
+      global: True
+      advanced: True
+      helpLink: postgres
+    ssl_ca_file:
+      description: Path (inside the container) to the CA bundle PostgreSQL uses to verify client certificates. Salt-managed.
+      global: True
+      advanced: True
+      helpLink: postgres
+    hba_file:
+      description: Path (inside the container) to the pg_hba.conf authentication file. Salt-managed — edit salt/postgres/files/pg_hba.conf.
+      global: True
+      advanced: True
+      helpLink: postgres
+    log_destination:
+      description: Where PostgreSQL writes its server log. 'stderr' routes to the container log stream.
+      global: True
+      advanced: True
+      helpLink: postgres
+    logging_collector:
+      description: Whether to run a separate logging collector process. Disabled because the docker log stream already captures stderr.
+      global: True
+      advanced: True
+      helpLink: postgres
+    shared_preload_libraries:
+      description: Comma-separated list of extensions loaded at server start. Required for pg_cron which drives pg_partman maintenance — do not remove.
+      global: True
+      advanced: True
+      helpLink: postgres
+    cron.database_name:
+      description: Database pg_cron schedules jobs in. Must be so_telegraf so partman maintenance runs in the right database context.
+      global: True
+      advanced: True
+      helpLink: postgres

salt/postgres/sostatus.sls

@@ -0,0 +1,21 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{% from 'allowed_states.map.jinja' import allowed_states %}
+{% if sls.split('.')[0] in allowed_states %}
+
+append_so-postgres_so-status.conf:
+  file.append:
+    - name: /opt/so/conf/so-status/so-status.conf
+    - text: so-postgres
+    - unless: grep -q so-postgres /opt/so/conf/so-status/so-status.conf
+
+{% else %}
+
+{{sls}}_state_not_allowed:
+  test.fail_without_changes:
+    - name: {{sls}}_state_not_allowed
+
+{% endif %}

salt/postgres/ssl.sls

@@ -0,0 +1,55 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{% from 'allowed_states.map.jinja' import allowed_states %}
+{% if sls.split('.')[0] in allowed_states %}
+{%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   from 'ca/map.jinja' import CA %}
+
+postgres_key:
+  x509.private_key_managed:
+    - name: /etc/pki/postgres.key
+    - keysize: 4096
+    - backup: True
+    - new: True
+    {% if salt['file.file_exists']('/etc/pki/postgres.key') -%}
+    - prereq:
+      - x509: /etc/pki/postgres.crt
+    {%- endif %}
+    - retry:
+        attempts: 5
+        interval: 30
+
+postgres_crt:
+  x509.certificate_managed:
+    - name: /etc/pki/postgres.crt
+    - ca_server: {{ CA.server }}
+    - subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }}
+    - signing_policy: postgres
+    - private_key: /etc/pki/postgres.key
+    - CN: {{ GLOBALS.hostname }}
+    - days_remaining: 7
+    - days_valid: 820
+    - backup: True
+    - timeout: 30
+    - retry:
+        attempts: 5
+        interval: 30
+
+postgresKeyperms:
+  file.managed:
+    - replace: False
+    - name: /etc/pki/postgres.key
+    - mode: 400
+    - user: 939
+    - group: 939
+
+{% else %}
+
+{{sls}}_state_not_allowed:
+  test.fail_without_changes:
+    - name: {{sls}}_state_not_allowed
+
+{% endif %}

salt/postgres/telegraf_users.sls

@@ -0,0 +1,157 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{% from 'allowed_states.map.jinja' import allowed_states %}
+{% if sls.split('.')[0] in allowed_states %}
+{%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   from 'telegraf/map.jinja' import TELEGRAFMERGED %}
+
+{# postgres_wait_ready below requires `docker_container: so-postgres`, which is
+   declared in postgres.enabled. Include it here so state.apply postgres.telegraf_users
+   on its own (e.g. from orch.deploy_newnode) still has that ID in scope. Salt
+   de-duplicates the circular include. #}
+include:
+  - postgres.enabled
+
+{% set TG_OUT = TELEGRAFMERGED.output | upper %}
+{% if TG_OUT in ['POSTGRES', 'BOTH'] %}
+
+# docker_container.running returns as soon as the container starts, but on
+# first-init docker-entrypoint.sh starts a temporary postgres with
+# `listen_addresses=''` to run /docker-entrypoint-initdb.d scripts, then
+# shuts it down before exec'ing the real CMD. A default pg_isready check
+# (Unix socket) passes during that ephemeral phase and races the shutdown
+# with "the database system is shutting down". Checking TCP readiness on
+# 127.0.0.1 only succeeds after the final postgres binds the port.
+postgres_wait_ready:
+  cmd.run:
+    - name: |
+        for i in $(seq 1 60); do
+          if docker exec so-postgres pg_isready -h 127.0.0.1 -U postgres -q 2>/dev/null; then
+            exit 0
+          fi
+          sleep 2
+        done
+        echo "so-postgres did not accept TCP connections within 120s" >&2
+        exit 1
+    - require:
+      - docker_container: so-postgres
+
+# Ensure the shared Telegraf database exists. init-users.sh only runs on a
+# fresh data dir, so hosts upgraded onto an existing /nsm/postgres volume
+# would otherwise never get so_telegraf.
+postgres_create_telegraf_db:
+  cmd.run:
+    - name: |
+        if ! docker exec so-postgres psql -U postgres -tAc "SELECT 1 FROM pg_database WHERE datname='so_telegraf'" | grep -q 1; then
+          docker exec so-postgres psql -v ON_ERROR_STOP=1 -U postgres -c "CREATE DATABASE so_telegraf"
+        fi
+    - require:
+      - cmd: postgres_wait_ready
+
+# Provision the shared group role and schema once. Every per-minion role is a
+# member of so_telegraf, and each Telegraf connection does SET ROLE so_telegraf
+# (via options='-c role=so_telegraf' in the connection string) so tables created
+# on first write are owned by the group role and every member can INSERT/SELECT.
+postgres_telegraf_group_role:
+  cmd.run:
+    - name: |
+        docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -U postgres -d so_telegraf <<'EOSQL'
+        DO $$
+        BEGIN
+            IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = 'so_telegraf') THEN
+                CREATE ROLE so_telegraf NOLOGIN;
+            END IF;
+        END
+        $$;
+        GRANT CONNECT ON DATABASE so_telegraf TO so_telegraf;
+        CREATE SCHEMA IF NOT EXISTS telegraf AUTHORIZATION so_telegraf;
+        GRANT USAGE, CREATE ON SCHEMA telegraf TO so_telegraf;
+        CREATE SCHEMA IF NOT EXISTS partman;
+        CREATE EXTENSION IF NOT EXISTS pg_partman SCHEMA partman;
+        CREATE EXTENSION IF NOT EXISTS pg_cron;
+        -- Telegraf (running as so_telegraf) calls partman.create_parent()
+        -- on first write of each metric, which needs USAGE on the partman
+        -- schema, EXECUTE on its functions/procedures, and write access to
+        -- partman.part_config so it can register new partitioned parents.
+        GRANT USAGE, CREATE ON SCHEMA partman TO so_telegraf;
+        GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA partman TO so_telegraf;
+        GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA partman TO so_telegraf;
+        GRANT EXECUTE ON ALL PROCEDURES IN SCHEMA partman TO so_telegraf;
+        -- partman creates per-parent template tables (partman.template_*) at
+        -- runtime; default privileges extend DML/sequence access to them.
+        ALTER DEFAULT PRIVILEGES IN SCHEMA partman
+            GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO so_telegraf;
+        ALTER DEFAULT PRIVILEGES IN SCHEMA partman
+            GRANT USAGE, SELECT, UPDATE ON SEQUENCES TO so_telegraf;
+        -- Hourly partman maintenance. cron.schedule is idempotent by jobname.
+        SELECT cron.schedule(
+          'telegraf-partman-maintenance',
+          '17 * * * *',
+          'CALL partman.run_maintenance_proc()'
+        );
+        EOSQL
+    - require:
+      - cmd: postgres_create_telegraf_db
+
+{%   set creds = salt['pillar.get']('telegraf:postgres_creds', {}) %}
+{%   for mid, entry in creds.items() %}
+{%     if entry.get('user') and entry.get('pass') %}
+{%       set u = entry.user %}
+{%       set p = entry.pass | replace("'", "''") %}
+
+postgres_telegraf_role_{{ u }}:
+  cmd.run:
+    - name: |
+        docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -U postgres -d so_telegraf <<'EOSQL'
+        DO $$
+        BEGIN
+            IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = '{{ u }}') THEN
+                EXECUTE format('CREATE ROLE %I WITH LOGIN PASSWORD %L', '{{ u }}', '{{ p }}');
+            ELSE
+                EXECUTE format('ALTER ROLE %I WITH PASSWORD %L', '{{ u }}', '{{ p }}');
+            END IF;
+        END
+        $$;
+        GRANT CONNECT ON DATABASE so_telegraf TO "{{ u }}";
+        GRANT so_telegraf TO "{{ u }}";
+        EOSQL
+    - require:
+      - cmd: postgres_telegraf_group_role
+
+{%     endif %}
+{%   endfor %}
+
+# Reconcile partman retention from pillar. Runs after role/schema setup so
+# any partitioned parents Telegraf has already created get their retention
+# refreshed whenever postgres.telegraf.retention_days changes.
+{%   set retention = salt['pillar.get']('postgres:telegraf:retention_days', 14) | int %}
+postgres_telegraf_retention_reconcile:
+  cmd.run:
+    - name: |
+        docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -U postgres -d so_telegraf <<'EOSQL'
+        DO $$
+        BEGIN
+            IF EXISTS (SELECT 1 FROM pg_catalog.pg_extension WHERE extname = 'pg_partman') THEN
+                UPDATE partman.part_config
+                SET retention = '{{ retention }} days',
+                    retention_keep_table = false
+                WHERE parent_table LIKE 'telegraf.%';
+            END IF;
+        END
+        $$;
+        EOSQL
+    - require:
+      - cmd: postgres_telegraf_group_role
+
+{% endif %}
+
+{% else %}
+
+{{sls}}_state_not_allowed:
+  test.fail_without_changes:
+    - name: {{sls}}_state_not_allowed
+
+{% endif %}

salt/postgres/tools/sbin/so-postgres-backup

@@ -0,0 +1,39 @@
+#!/bin/bash
+#
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+. /usr/sbin/so-common
+
+# Backups contain role password hashes and full chat data; keep them 0600.
+umask 0077
+
+TODAY=$(date '+%Y_%m_%d')
+BACKUPDIR=/nsm/backup
+BACKUPFILE="$BACKUPDIR/so-postgres-backup-$TODAY.sql.gz"
+MAXBACKUPS=7
+
+mkdir -p $BACKUPDIR
+
+# Skip if already backed up today
+if [ -f "$BACKUPFILE" ]; then
+  exit 0
+fi
+
+# Skip if container isn't running
+if ! docker ps --format '{{.Names}}' | grep -q '^so-postgres$'; then
+  exit 0
+fi
+
+# Dump all databases and roles, compress
+docker exec so-postgres pg_dumpall -U postgres | gzip > "$BACKUPFILE"
+
+# Retention cleanup
+NUMBACKUPS=$(find $BACKUPDIR -type f -name "so-postgres-backup*" | wc -l)
+while [ "$NUMBACKUPS" -gt "$MAXBACKUPS" ]; do
+  OLDEST=$(find $BACKUPDIR -type f -name "so-postgres-backup*" -printf '%T+ %p\n' | sort | head -n 1 | awk -F" " '{print $2}')
+  rm -f "$OLDEST"
+  NUMBACKUPS=$(find $BACKUPDIR -type f -name "so-postgres-backup*" | wc -l)
+done

salt/postgres/tools/sbin/so-postgres-manage

@@ -0,0 +1,80 @@
+#!/bin/bash
+
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+. /usr/sbin/so-common
+
+usage() {
+    echo "Usage: $0 <operation> [args]"
+    echo ""
+    echo "Supported Operations:"
+    echo "  sql          Execute a SQL command, requires: <sql>"
+    echo "  sqlfile      Execute a SQL file, requires: <path>"
+    echo "  shell        Open an interactive psql shell"
+    echo "  dblist       List databases"
+    echo "  userlist     List database roles"
+    echo ""
+    exit 1
+}
+
+if [ $# -lt 1 ]; then
+  usage
+fi
+
+# Check for prerequisites
+if [ "$(id -u)" -ne 0 ]; then
+    echo "This script must be run using sudo!"
+    exit 1
+fi
+
+COMMAND=$(basename $0)
+OP=$1
+shift
+
+set -eo pipefail
+
+log() {
+  echo -e "$(date) | $COMMAND | $@" >&2
+}
+
+so_psql() {
+  docker exec so-postgres psql -U postgres -d securityonion "$@"
+}
+
+case "$OP" in
+
+  sql)
+    [ $# -lt 1 ] && usage
+    so_psql -c "$1"
+    ;;
+
+  sqlfile)
+    [ $# -ne 1 ] && usage
+    if [ ! -f "$1" ]; then
+      log "File not found: $1"
+      exit 1
+    fi
+    docker cp "$1" so-postgres:/tmp/sqlfile.sql
+    docker exec so-postgres psql -U postgres -d securityonion -f /tmp/sqlfile.sql
+    docker exec so-postgres rm -f /tmp/sqlfile.sql
+    ;;
+
+  shell)
+    docker exec -it so-postgres psql -U postgres -d securityonion
+    ;;
+
+  dblist)
+    so_psql -c "\l"
+    ;;
+
+  userlist)
+    so_psql -c "\du"
+    ;;
+
+  *)
+    usage
+    ;;
+esac

salt/postgres/tools/sbin/so-postgres-restart

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+. /usr/sbin/so-common
+
+/usr/sbin/so-restart postgres $1

salt/postgres/tools/sbin/so-postgres-start

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+. /usr/sbin/so-common
+
+/usr/sbin/so-start postgres $1

salt/postgres/tools/sbin/so-postgres-stop

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+. /usr/sbin/so-common
+
+/usr/sbin/so-stop postgres $1

salt/postgres/tools/sbin/so-stats-show

@@ -0,0 +1,157 @@
+#!/bin/bash
+
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+# Point-in-time host metrics from the Telegraf Postgres backend.
+# Sanity-check tool for verifying metrics are landing before the grid
+# dashboards consume them.
+#
+# Assumes Telegraf's postgresql output is configured with
+# tags_as_foreign_keys = true, tags_as_jsonb = true, fields_as_jsonb = true,
+# so metric tables are (time, tag_id, fields jsonb) and tag tables are
+# (tag_id, tags jsonb).
+
+. /usr/sbin/so-common
+
+usage() {
+  cat <<EOF
+Usage: $0 [host]
+
+Shows the most recent CPU, memory, disk, and load metrics for each host
+from the so_telegraf Postgres database. Without an argument, reports on
+every host that has data. With a host, limits output to that one.
+
+Requires: sudo, so-postgres running, telegraf.output set to
+POSTGRES or BOTH.
+EOF
+  exit 1
+}
+
+if [ "$(id -u)" -ne 0 ]; then
+  echo "This script must be run using sudo!"
+  exit 1
+fi
+
+case "${1:-}" in
+  -h|--help) usage ;;
+esac
+
+FILTER_HOST="${1:-}"
+SCHEMA="telegraf"
+
+# Host values are interpolated into SQL below. Hostnames are [A-Za-z0-9._-];
+# any other character in a tag value or CLI arg is rejected to prevent a
+# stored-tag (or CLI) → SQL injection via a compromised Telegraf writer.
+HOST_RE='^[A-Za-z0-9._-]+$'
+if [ -n "$FILTER_HOST" ] && ! [[ "$FILTER_HOST" =~ $HOST_RE ]]; then
+  echo "Invalid host filter: $FILTER_HOST" >&2
+  exit 1
+fi
+
+so_psql() {
+  docker exec so-postgres psql -U postgres -d so_telegraf -At -F $'\t' "$@"
+}
+
+if ! docker exec so-postgres psql -U postgres -lqt 2>/dev/null | cut -d\| -f1 | grep -qw so_telegraf; then
+  echo "Database so_telegraf not found. Is telegraf.output set to POSTGRES or BOTH?"
+  exit 2
+fi
+
+table_exists() {
+  local table="$1"
+  [ -n "$(so_psql -c "SELECT 1 FROM information_schema.tables WHERE table_schema='${SCHEMA}' AND table_name='${table}' LIMIT 1;")" ]
+}
+
+# Discover hosts from cpu_tag (every minion reports cpu).
+if ! table_exists "cpu_tag"; then
+  echo "${SCHEMA}.cpu_tag not found. Has Telegraf written any rows yet?"
+  exit 0
+fi
+
+HOSTS=$(so_psql -c "
+  SELECT DISTINCT tags->>'host'
+  FROM \"${SCHEMA}\".cpu_tag
+  WHERE tags ? 'host'
+  ORDER BY 1;")
+
+if [ -z "$HOSTS" ]; then
+  echo "No hosts found in ${SCHEMA}. Is Telegraf configured to write to Postgres?"
+  exit 0
+fi
+
+print_metric() {
+  so_psql -c "$1"
+}
+
+for host in $HOSTS; do
+  if ! [[ "$host" =~ $HOST_RE ]]; then
+    echo "Skipping host with invalid characters in tag value: $host" >&2
+    continue
+  fi
+  if [ -n "$FILTER_HOST" ] && [ "$host" != "$FILTER_HOST" ]; then
+    continue
+  fi
+
+  echo "===================================================================="
+  echo " Host: $host"
+  echo "===================================================================="
+
+  if table_exists "cpu"; then
+    print_metric "
+      SELECT 'cpu          ' AS metric,
+             to_char(c.time, 'YYYY-MM-DD HH24:MI:SS') AS ts,
+             round((100 - (c.fields->>'usage_idle')::numeric), 1) || '% used'
+      FROM \"${SCHEMA}\".cpu c
+      JOIN \"${SCHEMA}\".cpu_tag t USING (tag_id)
+      WHERE t.tags->>'host' = '${host}' AND t.tags->>'cpu' = 'cpu-total'
+      ORDER BY c.time DESC LIMIT 1;"
+  fi
+
+  if table_exists "mem"; then
+    print_metric "
+      SELECT 'memory       ' AS metric,
+             to_char(m.time, 'YYYY-MM-DD HH24:MI:SS') AS ts,
+             round((m.fields->>'used_percent')::numeric, 1) || '% used (' ||
+             pg_size_pretty((m.fields->>'used')::bigint) || ' of ' ||
+             pg_size_pretty((m.fields->>'total')::bigint) || ')'
+      FROM \"${SCHEMA}\".mem m
+      JOIN \"${SCHEMA}\".mem_tag t USING (tag_id)
+      WHERE t.tags->>'host' = '${host}'
+      ORDER BY m.time DESC LIMIT 1;"
+  fi
+
+  if table_exists "disk"; then
+    print_metric "
+      SELECT 'disk ' || rpad(t.tags->>'path', 12) AS metric,
+             to_char(d.time, 'YYYY-MM-DD HH24:MI:SS') AS ts,
+             round((d.fields->>'used_percent')::numeric, 1) || '% used (' ||
+             pg_size_pretty((d.fields->>'used')::bigint) || ' of ' ||
+             pg_size_pretty((d.fields->>'total')::bigint) || ')'
+      FROM \"${SCHEMA}\".disk d
+      JOIN \"${SCHEMA}\".disk_tag t USING (tag_id)
+      WHERE t.tags->>'host' = '${host}'
+        AND d.time = (SELECT max(d2.time)
+                      FROM \"${SCHEMA}\".disk d2
+                      JOIN \"${SCHEMA}\".disk_tag t2 USING (tag_id)
+                      WHERE t2.tags->>'host' = '${host}')
+      ORDER BY t.tags->>'path';"
+  fi
+
+  if table_exists "system"; then
+    print_metric "
+      SELECT 'load         ' AS metric,
+             to_char(s.time, 'YYYY-MM-DD HH24:MI:SS') AS ts,
+             (s.fields->>'load1') || ' / ' ||
+             (s.fields->>'load5') || ' / ' ||
+             (s.fields->>'load15') || ' (1/5/15m)'
+      FROM \"${SCHEMA}\".system s
+      JOIN \"${SCHEMA}\".system_tag t USING (tag_id)
+      WHERE t.tags->>'host' = '${host}'
+      ORDER BY s.time DESC LIMIT 1;"
+  fi
+
+  echo ""
+done

salt/soc/defaults.map.jinja

@@ -24,6 +24,11 @@
 
 {% do SOCDEFAULTS.soc.config.server.modules.elastic.update({'username': GLOBALS.elasticsearch.auth.users.so_elastic_user.user, 'password': GLOBALS.elasticsearch.auth.users.so_elastic_user.pass}) %}
 
+{% if GLOBALS.postgres is defined and GLOBALS.postgres.auth is defined %}
+{%   set PG_ADMIN_PASS = salt['pillar.get']('secrets:postgres_pass', '') %}
+{% do SOCDEFAULTS.soc.config.server.modules.update({'postgres': {'hostUrl': GLOBALS.manager_ip, 'port': 5432, 'username': GLOBALS.postgres.auth.users.so_postgres_user.user, 'password': GLOBALS.postgres.auth.users.so_postgres_user.pass, 'adminUser': 'postgres', 'adminPassword': PG_ADMIN_PASS, 'dbname': 'securityonion', 'sslMode': 'require', 'assistantEnabled': true, 'esHostUrl': 'https://' ~ GLOBALS.manager_ip ~ ':9200', 'esUsername': GLOBALS.elasticsearch.auth.users.so_elastic_user.user, 'esPassword': GLOBALS.elasticsearch.auth.users.so_elastic_user.pass, 'esVerifyCert': false}}) %}
+{% endif %}
+
 {% do SOCDEFAULTS.soc.config.server.modules.influxdb.update({'hostUrl': 'https://' ~ GLOBALS.influxdb_host ~ ':8086'}) %}
 {% do SOCDEFAULTS.soc.config.server.modules.influxdb.update({'token': INFLUXDB_TOKEN}) %}
 {% for tool in SOCDEFAULTS.soc.config.server.client.tools %}

salt/soc/defaults.yaml

@@ -2622,6 +2622,7 @@ soc:
               This is a YARA rule template. Replace all template values with your own values.
               The YARA rule name is the unique identifier for the rule.
               Docs: https://yara.readthedocs.io/en/stable/writingrules.html#writing-yara-rules
+              Delete these comments before attempting to "Create" the rule
               */
 
               rule Example // This identifier _must_ be unique
@@ -2686,4 +2687,5 @@ soc:
               lowBalanceColorAlert: 500000
               enabled: true
               adapter: SOAI
+              charsPerTokenEstimate: 4
             

salt/soc/soc_soc.yaml

@@ -8,6 +8,7 @@ soc:
     description: When this setting is enabled and the grid is not in airgap mode, SOC will provide feature usage data to the Security Onion development team via Google Analytics. This data helps Security Onion developers determine which product features are being used and can also provide insight into improving the user interface. When changing this setting, wait for the grid to fully synchronize and then perform a hard browser refresh on SOC, to force the browser cache to update and reflect the new setting.
     global: True
     helpLink: telemetry
+    forcedType: bool
   files:
     soc:
       banner__md:
@@ -139,6 +140,7 @@ soc:
         title: Require TOTP
         description: Require all users to enable Time-based One Time Passwords (MFA) upon login to SOC.
         global: True
+        forcedType: bool
       customReportsPath:
         title: Custom Reports Path
         description: Path to custom markdown templates for PDF report generation. All markdown files in this directory will be available as custom reports in the SOC Reports interface.
@@ -185,6 +187,7 @@ soc:
         description: "Set to true to enable reverse DNS lookups for IP addresses in the SOC UI. To add your own local lookups, create a CSV file at /nsm/custom-mappings/ip-descriptions.csv on your Manager and populate the file with IP addresses and descriptions as follows: IP, Description. Elasticsearch will then ingest the CSV during the next high state."
         global: True
         helpLink: security-onion-console-customization#reverse-dns
+        forcedType: bool
       modules:
         elastalertengine:
           aiRepoUrl:
@@ -202,6 +205,7 @@ soc:
           showAiSummaries:
             description: Show AI summaries for ElastAlert rules.
             global: True
+            forcedType: bool
           additionalAlerters:
             title: "Notifications: Sev 0/Default Alerters"
             description: "Specify default alerters to enable for outbound notifications. These alerters will be used unless overridden by higher severity alerter settings. Specify one alerter name (Ex: 'email') per line. Alerters refers to ElastAlert 2 alerters, as documented at https://elastalert2.readthedocs.io. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key."
@@ -338,6 +342,7 @@ soc:
             description: 'Automatically update Sigma rules on a regular basis. This will update the rules based on the configured frequency.'
             global: True
             advanced: True
+            forcedType: bool
           communityRulesImportFrequencySeconds:
             description: 'How often to check for new Sigma rules (in seconds). This applies to both Community Rule Packages and any configured Git repos.'
             global: True
@@ -395,6 +400,7 @@ soc:
             description: Set to true if the SOC case management module, natively integrated with Elasticsearch, should be enabled.
             global: True
             advanced: True
+            forcedType: bool
           extractCommonObservables:
             description: List of indexed fields to automatically extract into a case observable, when attaching related events to a case.
             global: True
@@ -421,6 +427,7 @@ soc:
           lookupTunnelParent:
             description: When true, if a pivoted event appears to be encapsulated, such as in a VXLAN packet, then SOC will pivot to the VXLAN packet stream. When false, SOC will attempt to pivot to the encapsulated packet stream itself, but at the risk that it may be unable to locate it in the stored PCAP data.
             global: True
+            forcedType: bool
           maxScrollSize:
             description: The maximum number of documents to request in a single Elasticsearch scroll request.
           bulkIndexWorkerCount:
@@ -477,10 +484,12 @@ soc:
           showAiSummaries:
             description: Show AI summaries for Strelka rules.
             global: True
+            forcedType: bool
           autoUpdateEnabled:
             description: 'Automatically update YARA rules on a regular basis. This will update the rules based on the configured frequency.'
             global: True
             advanced: True
+            forcedType: bool
           autoEnabledYaraRules:
             description: 'YARA rules to automatically enable on initial import. Format is $Ruleset - for example, for the default shipped ruleset: securityonion-yara'
             global: True
@@ -536,10 +545,12 @@ soc:
           showAiSummaries:
             description: Show AI summaries for Suricata rules.
             global: True
+            forcedType: bool
           autoUpdateEnabled:
             description: 'Automatically update Suricata rules on a regular basis. This will update the rules based on the configured frequency.'
             global: True
             advanced: True
+            forcedType: bool
           communityRulesImportFrequencySeconds:
             description: 'How often to check for new Suricata rules (in seconds).'
             global: True
@@ -669,7 +680,7 @@ soc:
           adapters:
             description: Configuration for AI adapters used by the Onion AI assistant. Please see documentation for help on which fields are required for which protocols.
             global: True
-            advanced: True
+            advanced: False
             forcedType: "[]{}"
             helpLink: onion-ai
             syntax: json
@@ -709,6 +720,7 @@ soc:
           enabled:
             description: Set to true to enable the Onion AI assistant in SOC.
             global: True
+            forcedType: bool
           investigationPrompt: 
             description: Prompt given to Onion AI when beginning an investigation.
             global: True
@@ -734,7 +746,7 @@ soc:
           availableModels:
             description: List of AI models available for use in SOC as well as model specific warning thresholds.
             global: True
-            advanced: True
+            advanced: False
             forcedType: "[]{}"
             helpLink: onion-ai
             syntax: json
@@ -749,7 +761,7 @@ soc:
               required: True
             - field: origin
               label: Country of Origin for the Model Training
-              required: false
+              required: False
             - field: contextLimitSmall
               label: Context Limit (Small)
               forcedType: int
@@ -767,6 +779,10 @@ soc:
             - field: enabled
               label: Enabled
               forcedType: bool
+            - field: charsPerTokenEstimate
+              label: Characters per Token Estimate
+              forcedType: float
+              required: False
         apiTimeoutMs:
           description: Duration (in milliseconds) to wait for a response from the SOC server API before giving up and showing an error on the SOC UI.
           global: True
@@ -789,9 +805,11 @@ soc:
         casesEnabled:
           description: Set to true to enable case management in SOC.
           global: True
+          forcedType: bool
         detectionsEnabled:
           description: Set to true to enable the Detections module in SOC.
           global: True
+          forcedType: bool
         inactiveTools:
           description: List of external tools to remove from the SOC UI.
           global: True
@@ -867,6 +885,7 @@ soc:
           showUnreviewedAiSummaries:
             description: Show AI summaries in detections even if they have not yet been reviewed by a human.
             global: True
+            forcedType: bool
           templateDetections:
             suricata:
               description: The template used when creating a new Suricata detection. [publicId] will be replaced with an unused Public Id.
@@ -904,6 +923,7 @@ soc:
               customEnabled:
                 description: Set to true to allow users add their own artifact types directly in the SOC UI.
                 global: True
+                forcedType: bool
             category:
               labels:
                 description: List of available case categories.
@@ -911,6 +931,7 @@ soc:
               customEnabled:
                 description: Set to true to allow users add their own categories directly in the SOC UI.
                 global: True
+                forcedType: bool
             pap:
               labels:
                 description: List of available PAP (Permissible Actions Protocol) values.
@@ -918,6 +939,7 @@ soc:
               customEnabled:
                 description: Set to true to allow users add their own PAP values directly in the SOC UI.
                 global: True
+                forcedType: bool
             severity:
               labels:
                 description: List of available case severities.
@@ -925,6 +947,7 @@ soc:
               customEnabled:
                 description: Set to true to allow users add their own severities directly in the SOC UI.
                 global: True
+                forcedType: bool
             status:
               labels:
                 description: List of available case statuses. Note that some default statuses have special characteristics and related functionality built into SOC.
@@ -932,6 +955,7 @@ soc:
               customEnabled:
                 description: Set to true to allow users add their own case statuses directly in the SOC UI.
                 global: True
+                forcedType: bool
             tags:
               labels:
                 description: List of available tags.
@@ -939,6 +963,7 @@ soc:
               customEnabled:
                 description: Set to true to allow users add their own tags directly in the SOC UI.
                 global: True
+                forcedType: bool
             tlp:
               labels:
                 description: List of available TLP (Traffic Light Protocol) values.
@@ -946,3 +971,4 @@ soc:
               customEnabled:
                 description: Set to true to allow users add their own TLP values directly in the SOC UI.
                 global: True
+                forcedType: bool

salt/suricata/map.jinja

@@ -33,7 +33,7 @@
 {%   do SURICATAMERGED.config.outputs['pcap-log'].update({'conditional': SURICATAMERGED.pcap.conditional}) %}
 {%   do SURICATAMERGED.config.outputs['pcap-log'].update({'dir': SURICATAMERGED.pcap.dir}) %}
 {#   multiply maxsize by 1000 since it is saved in GB, i.e. 52 = 52000MB. filesize is also saved in MB and we strip the MB and convert to int #}
-{%   set maxfiles = (SURICATAMERGED.pcap.maxsize * 1000 / (SURICATAMERGED.pcap.filesize[:-2] | int) / SURICATAMERGED.config['af-packet'].threads | int) | round | int %}
+{%   set maxfiles = ([1, (SURICATAMERGED.pcap.maxsize * 1000 / (SURICATAMERGED.pcap.filesize[:-2] | int) / SURICATAMERGED.config['af-packet'].threads | int) | round(0, 'ceil') | int] | max) %}
 {%   do SURICATAMERGED.config.outputs['pcap-log'].update({'max-files': maxfiles}) %}
 {% endif %}
 

salt/suricata/soc_suricata.yaml

@@ -64,8 +64,10 @@ suricata:
       helpLink: suricata
     conditional: 
       description: Set to "all" to record PCAP for all flows. Set to "alerts" to only record PCAP for Suricata alerts. Set to "tag" to only record PCAP for tagged rules.
-      regex: ^(all|alerts|tag)$
-      regexFailureMessage: You must enter either all, alert or tag.
+      options:
+      - all
+      - alerts
+      - tag
       helpLink: suricata
     dir:
       description: Parent directory to store PCAP.
@@ -83,7 +85,9 @@ suricata:
         advanced: True
       cluster-type:
         advanced: True
-        regex: ^(cluster_flow|cluster_qm)$
+        options:
+        - cluster_flow
+        - cluster_qm
       defrag:
         description: Enable defragmentation of IP packets before processing.
         forcedType: bool
@@ -161,7 +165,7 @@ suricata:
       address-groups:
         HOME_NET: 
           description: Assign a list of hosts, or networks, using CIDR notation, to this Suricata variable. The variable can then be re-used within Suricata rules. This allows for a single adjustment to the variable that will then affect all rules referencing the variable.
-          regex: ^(((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\/([0-9]|[1-2][0-9]|3[0-2]))?$|^((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?))|:))|(([0-9A-Fa-f]{1,4}:){5}((:[0-9A-Fa-f]{1,4}){1,2}|:((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)|:))|(([0-9A-Fa-f]{1,4}:){4}((:[0-9A-Fa-f]{1,4}){1,3}|:((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)|:))|(([0-9A-Fa-f]{1,4}:){3}((:[0-9A-Fa-f]{1,4}){1,4}|:((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)|:))|(([0-9A-Fa-f]{1,4}:){2}((:[0-9A-Fa-f]{1,4}){1,5}|:((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)|:))|(([0-9A-Fa-f]{1,4}:){1}((:[0-9A-Fa-f]{1,4}){1,6}|:((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)|:))|(:((:[0-9A-Fa-f]{1,4}){1,7}|:)))(\/([0-9]|[1-9][0-9]|1[0-1][0-9]|12[0-8]))?$
+          regex: ^!?((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\/([0-9]|[1-2][0-9]|3[0-2]))?$|^!?((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)|:))|(([0-9A-Fa-f]{1,4}:){5}((:[0-9A-Fa-f]{1,4}){1,2}|:((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)|:))|(([0-9A-Fa-f]{1,4}:){4}((:[0-9A-Fa-f]{1,4}){1,3}|:((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)|:))|(([0-9A-Fa-f]{1,4}:){3}((:[0-9A-Fa-f]{1,4}){1,4}|:((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)|:))|(([0-9A-Fa-f]{1,4}:){2}((:[0-9A-Fa-f]{1,4}){1,5}|:((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)|:))|(([0-9A-Fa-f]{1,4}:){1}((:[0-9A-Fa-f]{1,4}){1,6}|:((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)|:))|(:((:[0-9A-Fa-f]{1,4}){1,7}|:)))(\/([0-9]|[1-9][0-9]|1[0-1][0-9]|12[0-8]))?$
           regexFailureMessage: You must enter a valid IP address or CIDR.
           forcedType: "[]string"
           duplicates: True

salt/suricata/tools/sbin_jinja/so-suricata-testrule

@@ -27,7 +27,7 @@ echo ""
 sleep 3
 
 rm -rf /tmp/nids-testing/output
-mkdir -p /tmp/nids-testing/output
+mkdir -p /tmp/nids-testing/output/suripcap
 chown suricata:socore /tmp/nids-testing/output
 mkdir -p /tmp/nids-testing/rules
 
@@ -45,7 +45,7 @@ echo "==== Begin Suricata Output ==="
     -v /opt/so/conf/suricata/bpf:/etc/suricata/bpf:ro \
     -v /tmp/nids-testing/output/:/nsm/:rw \
     {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-suricata:{{ VERSION }} \
-    --runmode single -v -k none -r /input.pcap -l /tmp --init-errors-fatal --set outputs.6.pcap-log.enabled=no
+    --runmode single -v -k none -r /input.pcap -l /tmp --init-errors-fatal
 echo "==== End Suricata Output ==="
 
 echo ""

salt/telegraf/defaults.yaml

@@ -1,5 +1,6 @@
 telegraf:
   enabled: False
+  output: BOTH
   config:
     interval: '30s'
     metric_batch_size: 1000

salt/telegraf/etc/telegraf.conf

@@ -8,6 +8,14 @@
 {%- set ZEEK_ENABLED = salt['pillar.get']('zeek:enabled', True) %}
 {%- set MDENGINE = GLOBALS.md_engine %}
 {%- set LOGSTASH_ENABLED = LOGSTASH_MERGED.enabled %}
+{%- set TG_OUT = TELEGRAFMERGED.output | upper %}
+{%- set PG_HOST = GLOBALS.manager_ip %}
+{#- Per-minion telegraf creds live in the grid-wide telegraf/creds.sls pillar,
+    written by /usr/sbin/so-telegraf-cred on the manager. Each minion looks up
+    its own entry by grains.id. #}
+{%- set PG_ENTRY = salt['pillar.get']('telegraf:postgres_creds:' ~ grains.id, {}) %}
+{%- set PG_USER = PG_ENTRY.get('user', '') %}
+{%- set PG_PASS = PG_ENTRY.get('pass', '') %}
 # Global tags can be specified here in key="value" format.
 [global_tags]
   role = "{{ GLOBALS.role.split('-') | last }}"
@@ -72,6 +80,7 @@
 #                            OUTPUT PLUGINS                                   #
 ###############################################################################
 
+{%- if TG_OUT in ['INFLUXDB', 'BOTH'] %}
 # Configuration for sending metrics to InfluxDB
 [[outputs.influxdb_v2]]
   urls = ["https://{{ INFLUXDBHOST }}:8086"]
@@ -85,6 +94,41 @@
    tls_key = "/etc/telegraf/telegraf.key"
   ## Use TLS but skip chain & host verification
   # insecure_skip_verify = false
+{%- endif %}
+
+{%- if TG_OUT in ['POSTGRES', 'BOTH'] and PG_USER and PG_PASS %}
+# Configuration for sending metrics to PostgreSQL.
+# options='-c role=so_telegraf' makes every connection SET ROLE to the shared
+# group role so tables created on first write are owned by so_telegraf, and
+# all per-minion members can INSERT/SELECT them via role inheritance.
+# fields_as_jsonb/tags_as_jsonb keep metric tables at a fixed column count so
+# high-cardinality inputs (docker, procstat, kafka) don't blow past the
+# Postgres 1600-column-per-table limit.
+[[outputs.postgresql]]
+  connection = "host={{ PG_HOST }} port=5432 user={{ PG_USER }} password={{ PG_PASS }} dbname=so_telegraf sslmode=verify-full sslrootcert=/etc/telegraf/ca.crt options='-c role=so_telegraf'"
+  schema = "telegraf"
+  tags_as_foreign_keys = true
+  tags_as_jsonb = true
+  fields_as_jsonb = true
+  # Every metric table is a daily time-range partitioned parent managed by
+  # pg_partman. Retention drops old partitions instead of row-by-row DELETEs.
+  {% raw %}
+  # pg_partman 5.x requires the control column (time) to be NOT NULL, so
+  # ALTER it before create_parent(). And create_parent() splits
+  # p_parent_table on '.' to look up raw identifiers, so the literal must
+  # be 'schema.name' (not '"schema"."name"' as .table|quoteLiteral emits).
+  # IF NOT EXISTS keeps the three templates idempotent so a Telegraf
+  # restart after any DB-side surgery re-runs them safely.
+  create_templates = [
+    '''CREATE TABLE IF NOT EXISTS {{ .table }} ({{ .columns }}) PARTITION BY RANGE ("time")''',
+    '''ALTER TABLE {{ .table }} ALTER COLUMN "time" SET NOT NULL''',
+    '''SELECT partman.create_parent(p_parent_table := {{ printf "%s.%s" .table.Schema .table.Name | quoteLiteral }}, p_control := 'time', p_type := 'range', p_interval := '1 day', p_premake := 3) WHERE NOT EXISTS (SELECT 1 FROM partman.part_config WHERE parent_table = {{ printf "%s.%s" .table.Schema .table.Name | quoteLiteral }})'''
+  ]
+  tag_table_create_templates = [
+    '''CREATE TABLE IF NOT EXISTS {{ .table }} ({{ .columns }}, PRIMARY KEY (tag_id))'''
+  ]
+  {% endraw %}
+{%- endif %}
 
 ###############################################################################
 #                            PROCESSOR PLUGINS                                #

salt/telegraf/soc_telegraf.yaml

@@ -4,6 +4,15 @@ telegraf:
     forcedType: bool
     advanced: True
     helpLink: influxdb
+  output:
+    description: Selects the backend(s) Telegraf writes metrics to. INFLUXDB keeps the current behavior; POSTGRES writes to the grid's Postgres instance; BOTH dual-writes for migration validation.
+    options:
+      - INFLUXDB
+      - POSTGRES
+      - BOTH
+    global: True
+    advanced: True
+    helpLink: influxdb
   config:
     interval:
       description: Data collection interval.

salt/top.sls

@@ -68,6 +68,7 @@ base:
     - backup.config_backup
     - nginx
     - influxdb
+    - postgres
     - soc
     - kratos
     - hydra
@@ -95,6 +96,7 @@ base:
     - backup.config_backup
     - nginx
     - influxdb
+    - postgres
     - soc
     - kratos
     - hydra
@@ -123,6 +125,7 @@ base:
     - registry
     - nginx
     - influxdb
+    - postgres
     - strelka.manager
     - soc
     - kratos
@@ -153,6 +156,7 @@ base:
     - registry
     - nginx
     - influxdb
+    - postgres
     - strelka.manager
     - soc
     - kratos
@@ -181,6 +185,7 @@ base:
     - manager
     - nginx
     - influxdb
+    - postgres
     - strelka.manager
     - soc
     - kratos

salt/vars/eval.map.jinja

@@ -1,4 +1,5 @@
 {% from 'vars/elasticsearch.map.jinja' import ELASTICSEARCH_GLOBALS %}
+{% from 'vars/postgres.map.jinja' import POSTGRES_GLOBALS %}
 {% from 'vars/sensor.map.jinja' import SENSOR_GLOBALS %}
 
 {% set ROLE_GLOBALS = {} %}
@@ -6,6 +7,7 @@
 {% set EVAL_GLOBALS =
    [
      ELASTICSEARCH_GLOBALS,
+     POSTGRES_GLOBALS,
      SENSOR_GLOBALS
    ]
 %}