Merge pull request #15732 from Security-Onion-Solutions/hotfix/2.4.211

Hotfix 2.4.211
Merge pull request #15731 from Security-Onion-Solutions/hf211
2026-05-06 11:28:46 +02:00 · 2026-04-07 13:19:57 -04:00 · 2026-04-07 11:38:52 -04:00 · 2026-04-07 11:27:57 -04:00 · 2026-04-06 14:29:00 -04:00 · 2026-04-06 14:26:25 -04:00
398 changed files with 49371 additions and 83003 deletions
@@ -0,0 +1,546 @@
+title = "gitleaks config"
+
+# Gitleaks rules are defined by regular expressions and entropy ranges.
+# Some secrets have unique signatures which make detecting those secrets easy.
+# Examples of those secrets would be GitLab Personal Access Tokens, AWS keys, and GitHub Access Tokens.
+# All these examples have defined prefixes like `glpat`, `AKIA`, `ghp_`, etc.
+#
+# Other secrets might just be a hash which means we need to write more complex rules to verify
+# that what we are matching is a secret.
+#
+# Here is an example of a semi-generic secret
+#
+#   discord_client_secret = "8dyfuiRyq=vVc3RRr_edRk-fK__JItpZ"
+#
+# We can write a regular expression to capture the variable name (identifier),
+# the assignment symbol (like '=' or ':='), and finally the actual secret.
+# The structure of a rule to match this example secret is below:
+#
+#                                                           Beginning string
+#                                                               quotation
+#                                                                   │            End string quotation
+#                                                                   │                      │
+#                                                                   ▼                      ▼
+#    (?i)(discord[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9=_\-]{32})['\"]
+#
+#                   ▲                              ▲                                ▲
+#                   │                              │                                │
+#                   │                              │                                │
+#              identifier                  assignment symbol
+#                                                                                Secret
+#
+[[rules]]
+id = "gitlab-pat"
+description = "GitLab Personal Access Token"
+regex = '''glpat-[0-9a-zA-Z\-\_]{20}'''
+
+[[rules]]
+id = "aws-access-token"
+description = "AWS"
+regex = '''(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}'''
+
+# Cryptographic keys
+[[rules]]
+id = "PKCS8-PK"
+description = "PKCS8 private key"
+regex = '''-----BEGIN PRIVATE KEY-----'''
+
+[[rules]]
+id = "RSA-PK"
+description = "RSA private key"
+regex = '''-----BEGIN RSA PRIVATE KEY-----'''
+
+[[rules]]
+id = "OPENSSH-PK"
+description = "SSH private key"
+regex = '''-----BEGIN OPENSSH PRIVATE KEY-----'''
+
+[[rules]]
+id = "PGP-PK"
+description = "PGP private key"
+regex = '''-----BEGIN PGP PRIVATE KEY BLOCK-----'''
+
+[[rules]]
+id = "github-pat"
+description = "GitHub Personal Access Token"
+regex = '''ghp_[0-9a-zA-Z]{36}'''
+
+[[rules]]
+id = "github-oauth"
+description = "GitHub OAuth Access Token"
+regex = '''gho_[0-9a-zA-Z]{36}'''
+
+[[rules]]
+id = "SSH-DSA-PK"
+description = "SSH (DSA) private key"
+regex = '''-----BEGIN DSA PRIVATE KEY-----'''
+
+[[rules]]
+id = "SSH-EC-PK"
+description = "SSH (EC) private key"
+regex = '''-----BEGIN EC PRIVATE KEY-----'''
+
+
+[[rules]]
+id = "github-app-token"
+description = "GitHub App Token"
+regex = '''(ghu|ghs)_[0-9a-zA-Z]{36}'''
+
+[[rules]]
+id = "github-refresh-token"
+description = "GitHub Refresh Token"
+regex = '''ghr_[0-9a-zA-Z]{76}'''
+
+[[rules]]
+id = "shopify-shared-secret"
+description = "Shopify shared secret"
+regex = '''shpss_[a-fA-F0-9]{32}'''
+
+[[rules]]
+id = "shopify-access-token"
+description = "Shopify access token"
+regex = '''shpat_[a-fA-F0-9]{32}'''
+
+[[rules]]
+id = "shopify-custom-access-token"
+description = "Shopify custom app access token"
+regex = '''shpca_[a-fA-F0-9]{32}'''
+
+[[rules]]
+id = "shopify-private-app-access-token"
+description = "Shopify private app access token"
+regex = '''shppa_[a-fA-F0-9]{32}'''
+
+[[rules]]
+id = "slack-access-token"
+description = "Slack token"
+regex = '''xox[baprs]-([0-9a-zA-Z]{10,48})?'''
+
+[[rules]]
+id = "stripe-access-token"
+description = "Stripe"
+regex = '''(?i)(sk|pk)_(test|live)_[0-9a-z]{10,32}'''
+
+[[rules]]
+id = "pypi-upload-token"
+description = "PyPI upload token"
+regex = '''pypi-AgEIcHlwaS5vcmc[A-Za-z0-9\-_]{50,1000}'''
+
+[[rules]]
+id = "gcp-service-account"
+description = "Google (GCP) Service-account"
+regex = '''\"type\": \"service_account\"'''
+
+[[rules]]
+id = "heroku-api-key"
+description = "Heroku API Key"
+regex = ''' (?i)(heroku[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([0-9A-F]{8}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{12})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "slack-web-hook"
+description = "Slack Webhook"
+regex = '''https://hooks.slack.com/services/T[a-zA-Z0-9_]{8}/B[a-zA-Z0-9_]{8,12}/[a-zA-Z0-9_]{24}'''
+
+[[rules]]
+id = "twilio-api-key"
+description = "Twilio API Key"
+regex = '''SK[0-9a-fA-F]{32}'''
+
+[[rules]]
+id = "age-secret-key"
+description = "Age secret key"
+regex = '''AGE-SECRET-KEY-1[QPZRY9X8GF2TVDW0S3JN54KHCE6MUA7L]{58}'''
+
+[[rules]]
+id = "facebook-token"
+description = "Facebook token"
+regex = '''(?i)(facebook[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-f0-9]{32})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "twitter-token"
+description = "Twitter token"
+regex = '''(?i)(twitter[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-f0-9]{35,44})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "adobe-client-id"
+description = "Adobe Client ID (Oauth Web)"
+regex = '''(?i)(adobe[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-f0-9]{32})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "adobe-client-secret"
+description = "Adobe Client Secret"
+regex = '''(p8e-)(?i)[a-z0-9]{32}'''
+
+[[rules]]
+id = "alibaba-access-key-id"
+description = "Alibaba AccessKey ID"
+regex = '''(LTAI)(?i)[a-z0-9]{20}'''
+
+[[rules]]
+id = "alibaba-secret-key"
+description = "Alibaba Secret Key"
+regex = '''(?i)(alibaba[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{30})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "asana-client-id"
+description = "Asana Client ID"
+regex = '''(?i)(asana[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([0-9]{16})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "asana-client-secret"
+description = "Asana Client Secret"
+regex = '''(?i)(asana[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{32})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "atlassian-api-token"
+description = "Atlassian API token"
+regex = '''(?i)(atlassian[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{24})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "bitbucket-client-id"
+description = "Bitbucket client ID"
+regex = '''(?i)(bitbucket[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{32})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "bitbucket-client-secret"
+description = "Bitbucket client secret"
+regex = '''(?i)(bitbucket[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9_\-]{64})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "beamer-api-token"
+description = "Beamer API token"
+regex = '''(?i)(beamer[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](b_[a-z0-9=_\-]{44})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "clojars-api-token"
+description = "Clojars API token"
+regex = '''(CLOJARS_)(?i)[a-z0-9]{60}'''
+
+[[rules]]
+id = "contentful-delivery-api-token"
+description = "Contentful delivery API token"
+regex = '''(?i)(contentful[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9\-=_]{43})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "databricks-api-token"
+description = "Databricks API token"
+regex = '''dapi[a-h0-9]{32}'''
+
+[[rules]]
+id = "discord-api-token"
+description = "Discord API key"
+regex = '''(?i)(discord[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-h0-9]{64})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "discord-client-id"
+description = "Discord client ID"
+regex = '''(?i)(discord[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([0-9]{18})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "discord-client-secret"
+description = "Discord client secret"
+regex = '''(?i)(discord[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9=_\-]{32})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "doppler-api-token"
+description = "Doppler API token"
+regex = '''['\"](dp\.pt\.)(?i)[a-z0-9]{43}['\"]'''
+
+[[rules]]
+id = "dropbox-api-secret"
+description = "Dropbox API secret/key"
+regex = '''(?i)(dropbox[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{15})['\"]'''
+
+[[rules]]
+id = "dropbox--api-key"
+description = "Dropbox API secret/key"
+regex = '''(?i)(dropbox[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{15})['\"]'''
+
+[[rules]]
+id = "dropbox-short-lived-api-token"
+description = "Dropbox short lived API token"
+regex = '''(?i)(dropbox[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](sl\.[a-z0-9\-=_]{135})['\"]'''
+
+[[rules]]
+id = "dropbox-long-lived-api-token"
+description = "Dropbox long lived API token"
+regex = '''(?i)(dropbox[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"][a-z0-9]{11}(AAAAAAAAAA)[a-z0-9\-_=]{43}['\"]'''
+
+[[rules]]
+id = "duffel-api-token"
+description = "Duffel API token"
+regex = '''['\"]duffel_(test|live)_(?i)[a-z0-9_-]{43}['\"]'''
+
+[[rules]]
+id = "dynatrace-api-token"
+description = "Dynatrace API token"
+regex = '''['\"]dt0c01\.(?i)[a-z0-9]{24}\.[a-z0-9]{64}['\"]'''
+
+[[rules]]
+id = "easypost-api-token"
+description = "EasyPost API token"
+regex = '''['\"]EZAK(?i)[a-z0-9]{54}['\"]'''
+
+[[rules]]
+id = "easypost-test-api-token"
+description = "EasyPost test API token"
+regex = '''['\"]EZTK(?i)[a-z0-9]{54}['\"]'''
+
+[[rules]]
+id = "fastly-api-token"
+description = "Fastly API token"
+regex = '''(?i)(fastly[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9\-=_]{32})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "finicity-client-secret"
+description = "Finicity client secret"
+regex = '''(?i)(finicity[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{20})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "finicity-api-token"
+description = "Finicity API token"
+regex = '''(?i)(finicity[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-f0-9]{32})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "flutterwave-public-key"
+description = "Flutterwave public key"
+regex = '''FLWPUBK_TEST-(?i)[a-h0-9]{32}-X'''
+
+[[rules]]
+id = "flutterwave-secret-key"
+description = "Flutterwave secret key"
+regex = '''FLWSECK_TEST-(?i)[a-h0-9]{32}-X'''
+
+[[rules]]
+id = "flutterwave-enc-key"
+description = "Flutterwave encrypted key"
+regex = '''FLWSECK_TEST[a-h0-9]{12}'''
+
+[[rules]]
+id = "frameio-api-token"
+description = "Frame.io API token"
+regex = '''fio-u-(?i)[a-z0-9\-_=]{64}'''
+
+[[rules]]
+id = "gocardless-api-token"
+description = "GoCardless API token"
+regex = '''['\"]live_(?i)[a-z0-9\-_=]{40}['\"]'''
+
+[[rules]]
+id = "grafana-api-token"
+description = "Grafana API token"
+regex = '''['\"]eyJrIjoi(?i)[a-z0-9\-_=]{72,92}['\"]'''
+
+[[rules]]
+id = "hashicorp-tf-api-token"
+description = "HashiCorp Terraform user/org API token"
+regex = '''['\"](?i)[a-z0-9]{14}\.atlasv1\.[a-z0-9\-_=]{60,70}['\"]'''
+
+[[rules]]
+id = "hubspot-api-token"
+description = "HubSpot API token"
+regex = '''(?i)(hubspot[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-h0-9]{8}-[a-h0-9]{4}-[a-h0-9]{4}-[a-h0-9]{4}-[a-h0-9]{12})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "intercom-api-token"
+description = "Intercom API token"
+regex = '''(?i)(intercom[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9=_]{60})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "intercom-client-secret"
+description = "Intercom client secret/ID"
+regex = '''(?i)(intercom[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-h0-9]{8}-[a-h0-9]{4}-[a-h0-9]{4}-[a-h0-9]{4}-[a-h0-9]{12})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "ionic-api-token"
+description = "Ionic API token"
+regex = '''(?i)(ionic[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](ion_[a-z0-9]{42})['\"]'''
+
+[[rules]]
+id = "linear-api-token"
+description = "Linear API token"
+regex = '''lin_api_(?i)[a-z0-9]{40}'''
+
+[[rules]]
+id = "linear-client-secret"
+description = "Linear client secret/ID"
+regex = '''(?i)(linear[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-f0-9]{32})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "lob-api-key"
+description = "Lob API Key"
+regex = '''(?i)(lob[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]((live|test)_[a-f0-9]{35})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "lob-pub-api-key"
+description = "Lob Publishable API Key"
+regex = '''(?i)(lob[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]((test|live)_pub_[a-f0-9]{31})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "mailchimp-api-key"
+description = "Mailchimp API key"
+regex = '''(?i)(mailchimp[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-f0-9]{32}-us20)['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "mailgun-private-api-token"
+description = "Mailgun private API token"
+regex = '''(?i)(mailgun[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](key-[a-f0-9]{32})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "mailgun-pub-key"
+description = "Mailgun public validation key"
+regex = '''(?i)(mailgun[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](pubkey-[a-f0-9]{32})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "mailgun-signing-key"
+description = "Mailgun webhook signing key"
+regex = '''(?i)(mailgun[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-h0-9]{32}-[a-h0-9]{8}-[a-h0-9]{8})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "mapbox-api-token"
+description = "Mapbox API token"
+regex = '''(?i)(pk\.[a-z0-9]{60}\.[a-z0-9]{22})'''
+
+[[rules]]
+id = "messagebird-api-token"
+description = "MessageBird API token"
+regex = '''(?i)(messagebird[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{25})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "messagebird-client-id"
+description = "MessageBird API client ID"
+regex = '''(?i)(messagebird[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-h0-9]{8}-[a-h0-9]{4}-[a-h0-9]{4}-[a-h0-9]{4}-[a-h0-9]{12})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "new-relic-user-api-key"
+description = "New Relic user API Key"
+regex = '''['\"](NRAK-[A-Z0-9]{27})['\"]'''
+
+[[rules]]
+id = "new-relic-user-api-id"
+description = "New Relic user API ID"
+regex = '''(?i)(newrelic[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([A-Z0-9]{64})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "new-relic-browser-api-token"
+description = "New Relic ingest browser API token"
+regex = '''['\"](NRJS-[a-f0-9]{19})['\"]'''
+
+[[rules]]
+id = "npm-access-token"
+description = "npm access token"
+regex = '''['\"](npm_(?i)[a-z0-9]{36})['\"]'''
+
+[[rules]]
+id = "planetscale-password"
+description = "PlanetScale password"
+regex = '''pscale_pw_(?i)[a-z0-9\-_\.]{43}'''
+
+[[rules]]
+id = "planetscale-api-token"
+description = "PlanetScale API token"
+regex = '''pscale_tkn_(?i)[a-z0-9\-_\.]{43}'''
+
+[[rules]]
+id = "postman-api-token"
+description = "Postman API token"
+regex = '''PMAK-(?i)[a-f0-9]{24}\-[a-f0-9]{34}'''
+
+[[rules]]
+id = "pulumi-api-token"
+description = "Pulumi API token"
+regex = '''pul-[a-f0-9]{40}'''
+
+[[rules]]
+id = "rubygems-api-token"
+description = "Rubygem API token"
+regex = '''rubygems_[a-f0-9]{48}'''
+
+[[rules]]
+id = "sendgrid-api-token"
+description = "SendGrid API token"
+regex = '''SG\.(?i)[a-z0-9_\-\.]{66}'''
+
+[[rules]]
+id = "sendinblue-api-token"
+description = "Sendinblue API token"
+regex = '''xkeysib-[a-f0-9]{64}\-(?i)[a-z0-9]{16}'''
+
+[[rules]]
+id = "shippo-api-token"
+description = "Shippo API token"
+regex = '''shippo_(live|test)_[a-f0-9]{40}'''
+
+[[rules]]
+id = "linkedin-client-secret"
+description = "LinkedIn Client secret"
+regex = '''(?i)(linkedin[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z]{16})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "linkedin-client-id"
+description = "LinkedIn Client ID"
+regex = '''(?i)(linkedin[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{14})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "twitch-api-token"
+description = "Twitch API token"
+regex = '''(?i)(twitch[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{30})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "typeform-api-token"
+description = "Typeform API token"
+regex = '''(?i)(typeform[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}(tfp_[a-z0-9\-_\.=]{59})'''
+secretGroup = 3
+
+[[rules]]
+id = "generic-api-key"
+description = "Generic API Key"
+regex = '''(?i)((key|api[^Version]|token|secret|password)[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([0-9a-zA-Z\-_=]{8,64})['\"]'''
+entropy = 3.7
+secretGroup = 4
+
+
+[allowlist]
+description = "global allow lists"
+regexes = ['''219-09-9999''', '''078-05-1120''', '''(9[0-9]{2}|666)-\d{2}-\d{4}''', '''RPM-GPG-KEY.*''', '''.*:.*StrelkaHexDump.*''', '''.*:.*PLACEHOLDER.*''', '''ssl_.*password''', '''integration_key\s=\s"so-logs-"''']
+paths = [
+    '''gitleaks.toml''',
+    '''(.*?)(jpg|gif|doc|pdf|bin|svg|socket)$''',
+    '''(go.mod|go.sum)$''',
+    '''salt/nginx/files/enterprise-attack.json''',
+    '''(.*?)whl$'''
+]
@@ -2,11 +2,13 @@ body:
  - type: markdown
    attributes:
      value: |
+        ⚠️ This category is solely for conversations related to Security Onion 2.4 ⚠️
+
        If your organization needs more immediate, enterprise grade professional support, with one-on-one virtual meetings and screensharing, contact us via our website: https://securityonion.com/support
  - type: dropdown
    attributes:
      label: Version
-      description: Which version of Security Onion are you asking about?
+      description: Which version of Security Onion 2.4.x are you asking about?
      options:
        -
        - 2.4.10
@@ -1,178 +0,0 @@
-body:
-  - type: markdown
-    attributes:
-      value: |
-        If your organization needs more immediate, enterprise grade professional support, with one-on-one virtual meetings and screensharing, contact us via our website: https://securityonion.com/support
-  - type: dropdown
-    attributes:
-      label: Version
-      description: Which version of Security Onion are you asking about?
-      options:
-        -
-        - 3.0.0
-        - 3.1.0
-        - Other (please provide detail below)
-    validations:
-      required: true
-  - type: dropdown
-    attributes:
-      label: Installation Method
-      description: How did you install Security Onion?
-      options:
-        -
-        - Security Onion ISO image
-        - Cloud image (Amazon, Azure, Google)
-        - Network installation on Oracle 9 (unsupported)
-        - Other (please provide detail below)
-    validations:
-      required: true
-  - type: dropdown
-    attributes:
-      label: Description
-      description: >
-        Is this discussion about installation, configuration, upgrading, or other?
-      options:
-        -
-        - installation
-        - configuration
-        - upgrading
-        - other (please provide detail below)
-    validations:
-      required: true
-  - type: dropdown
-    attributes:
-      label: Installation Type
-      description: >
-        When you installed, did you choose Import, Eval, Standalone, Distributed, or something else?
-      options:
-        -
-        - Import
-        - Eval
-        - Standalone
-        - Distributed
-        - other (please provide detail below)
-    validations:
-      required: true
-  - type: dropdown
-    attributes:
-      label: Location
-      description: >
-        Is this deployment in the cloud, on-prem with Internet access, or airgap?
-      options:
-        -
-        - cloud
-        - on-prem with Internet access
-        - airgap
-        - other (please provide detail below)
-    validations:
-      required: true
-  - type: dropdown
-    attributes:
-      label: Hardware Specs
-      description: >
-        Does your hardware meet or exceed the minimum requirements for your installation type as shown at https://securityonion.net/docs/hardware?
-      options:
-        -
-        - Meets minimum requirements
-        - Exceeds minimum requirements
-        - Does not meet minimum requirements
-        - other (please provide detail below)
-    validations:
-      required: true
-  - type: input
-    attributes:
-      label: CPU
-      description: How many CPU cores do you have?
-    validations:
-      required: true
-  - type: input
-    attributes:
-      label: RAM
-      description: How much RAM do you have?
-    validations:
-      required: true
-  - type: input
-    attributes:
-      label: Storage for /
-      description: How much storage do you have for the / partition?
-    validations:
-      required: true
-  - type: input
-    attributes:
-      label: Storage for /nsm
-      description: How much storage do you have for the /nsm partition?
-    validations:
-      required: true
-  - type: dropdown
-    attributes:
-      label: Network Traffic Collection
-      description: >
-        Are you collecting network traffic from a tap or span port?
-      options:
-        -
-        - tap
-        - span port
-        - other (please provide detail below)
-    validations:
-      required: true
-  - type: dropdown
-    attributes:
-      label: Network Traffic Speeds
-      description: >
-        How much network traffic are you monitoring?
-      options:
-        -
-        - Less than 1Gbps
-        - 1Gbps to 10Gbps
-        - more than 10Gbps
-    validations:
-      required: true
-  - type: dropdown
-    attributes:
-      label: Status
-      description: >
-        Does SOC Grid show all services on all nodes as running OK?
-      options:
-        -
-        - Yes, all services on all nodes are running OK
-        - No, one or more services are failed (please provide detail below)
-    validations:
-      required: true
-  - type: dropdown
-    attributes:
-      label: Salt Status
-      description: >
-        Do you get any failures when you run "sudo salt-call state.highstate"?
-      options:
-        -
-        - Yes, there are salt failures (please provide detail below)
-        - No, there are no failures
-    validations:
-      required: true
-  - type: dropdown
-    attributes:
-      label: Logs
-      description: >
-        Are there any additional clues in /opt/so/log/?
-      options:
-        -
-        - Yes, there are additional clues in /opt/so/log/ (please provide detail below)
-        - No, there are no additional clues
-    validations:
-      required: true
-  - type: textarea
-    attributes:
-      label: Detail
-      description: Please read our discussion guidelines at https://github.com/Security-Onion-Solutions/securityonion/discussions/1720 and then provide detailed information to help us help you.
-      placeholder: |-
-        STOP! Before typing, please read our discussion guidelines at https://github.com/Security-Onion-Solutions/securityonion/discussions/1720 in their entirety!
-
-        If your organization needs more immediate, enterprise grade professional support, with one-on-one virtual meetings and screensharing, contact us via our website: https://securityonion.com/support
-    validations:
-      required: true
-  - type: checkboxes
-    attributes:
-      label: Guidelines
-      options:
-        - label: I have read the discussion guidelines at https://github.com/Security-Onion-Solutions/securityonion/discussions/1720 and assert that I have followed the guidelines.
-          required: true
@@ -1,22 +0,0 @@
-## Description
-
-<!-- 
-Explain the purpose of the pull request. Be brief or detailed depending on the scope of the changes.
-->
-
-## Related Issues
-
-<!-- 
-Optionally, list any related issues that this pull request addresses.
-->
-
-## Checklist
-
- [ ] I have read and followed the [CONTRIBUTING.md](https://github.com/Security-Onion-Solutions/securityonion/blob/3/main/CONTRIBUTING.md) file.
- [ ] I have read and agree to the terms of the [Contributor License Agreement](https://securityonionsolutions.com/cla)
-
-## Questions or Comments
-
-<!--
-If you have any questions or comments about this pull request, add them here.
-->
@@ -0,0 +1,24 @@
+name: contrib
+on:
+  issue_comment:
+    types: [created]
+  pull_request_target:
+    types: [opened,closed,synchronize]
+
+jobs:
+  CLAssistant:
+    runs-on: ubuntu-latest
+    steps:
+      - name: "Contributor Check"
+        if: (github.event.comment.body == 'recheck' || github.event.comment.body == 'I have read the CLA Document and I hereby sign the CLA') || github.event_name == 'pull_request_target'
+        uses: cla-assistant/github-action@v2.3.1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          PERSONAL_ACCESS_TOKEN : ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+        with:
+          path-to-signatures: 'signatures_v1.json'
+          path-to-document: 'https://securityonionsolutions.com/cla' 
+          allowlist: dependabot[bot],jertel,dougburks,TOoSmOotH,defensivedepth,m0duspwnens
+          remote-organization-name: Security-Onion-Solutions
+          remote-repository-name: licensing
+
@@ -0,0 +1,17 @@
+name: leak-test
+
+on: [pull_request]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        fetch-depth: '0'
+
+    - name: Gitleaks
+      uses: gitleaks/gitleaks-action@v1.6.0
+      with:
+        config-path: .github/.gitleaks.toml
@@ -13,7 +13,7 @@ jobs:
    strategy:
      fail-fast: false
      matrix:
-        python-version: ["3.14"]
+        python-version: ["3.13"]
        python-code-path: ["salt/sensoroni/files/analyzers", "salt/manager/tools/sbin"]

    steps:
@@ -23,7 +23,7 @@

 * Link the PR to the related issue, either using [keywords](https://docs.github.com/en/issues/tracking-your-work-with-issues/creating-issues/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword) in the PR description, or [manually](https://docs.github.com/en/issues/tracking-your-work-with-issues/creating-issues/linking-a-pull-request-to-an-issue#manually-linking-a-pull-request-to-an-issue).

-* **Pull requests should be opened against the current `?/dev` branch of this repo**, and should clearly describe the problem and solution.
+* **Pull requests should be opened against the `dev` branch of this repo**, and should clearly describe the problem and solution.

 * Be sure you have tested your changes and are confident they will not break other parts of the product.

@@ -1,46 +1,46 @@
-### 3.0.0-20260331 ISO image released on 2026/03/31
+### 2.4.211-20260407 ISO image released on 2026/04/07


 ### Download and Verify

-3.0.0-20260331 ISO image:  
-https://download.securityonion.net/file/securityonion/securityonion-3.0.0-20260331.iso
+2.4.211-20260407 ISO image:  
+https://download.securityonion.net/file/securityonion/securityonion-2.4.211-20260407.iso
 
-MD5: ECD318A1662A6FDE0EF213F5A9BD4B07  
-SHA1: E55BE314440CCF3392DC0B06BC5E270B43176D9C  
-SHA256: 7FC47405E335CBE5C2B6C51FE7AC60248F35CBE504907B8B5A33822B23F8F4D5  
+MD5: 35ECDD0BC10E56874D9F5725CA6C5888  
+SHA1: 30CE6CB0ED0059A3260368E4F296B8DBA381F9CD  
+SHA256: 185D8CF49CD3BFDD8876B8DDE48343DA90804B0C0EC3EADF0AD90D29C55E72B7  

 Signature for ISO image:  
-https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.0.0-20260331.iso.sig
+https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.211-20260407.iso.sig

 Signing key:  
-https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/3/main/KEYS  
+https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.4/main/KEYS  

 For example, here are the steps you can use on most Linux distributions to download and verify our Security Onion ISO image.

 Download and import the signing key:  
 ```
-wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/3/main/KEYS -O - | gpg --import -  
+wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.4/main/KEYS -O - | gpg --import -  
 ```

 Download the signature file for the ISO:  
 ```
-wget https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.0.0-20260331.iso.sig
+wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.211-20260407.iso.sig
 ```

 Download the ISO image:  
 ```
-wget https://download.securityonion.net/file/securityonion/securityonion-3.0.0-20260331.iso
+wget https://download.securityonion.net/file/securityonion/securityonion-2.4.211-20260407.iso
 ```

 Verify the downloaded ISO image using the signature file:  
 ```
-gpg --verify securityonion-3.0.0-20260331.iso.sig securityonion-3.0.0-20260331.iso
+gpg --verify securityonion-2.4.211-20260407.iso.sig securityonion-2.4.211-20260407.iso
 ```

 The output should show "Good signature" and the Primary key fingerprint should match what's shown below:
 ```
-gpg: Signature made Mon 30 Mar 2026 06:22:14 PM EDT using RSA key ID FE507013
+gpg: Signature made Mon 06 Apr 2026 02:58:51 PM EDT using RSA key ID FE507013
 gpg: Good signature from "Security Onion Solutions, LLC <info@securityonionsolutions.com>"
 gpg: WARNING: This key is not certified with a trusted signature!
 gpg:          There is no indication that the signature belongs to the owner.
@@ -0,0 +1 @@
+20260407
@@ -1,58 +1,50 @@
-<p align="center">
-  <img src="https://securityonionsolutions.com/logo/logo-so-onion-dark.svg" width="400" alt="Security Onion Logo">
-</p>
+## Security Onion 2.4

-# Security Onion
+Security Onion 2.4 is here!

-Security Onion is a free and open Linux distribution for threat hunting, enterprise security monitoring, and log management. It includes a comprehensive suite of tools designed to work together to provide visibility into your network and host activity.
+## Screenshots

-## ✨ Features
+Alerts
+![Alerts](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/50_alerts.png)

-Security Onion includes everything you need to monitor your network and host systems:
+Dashboards
+![Dashboards](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/53_dashboards.png)

-*   **Security Onion Console (SOC)**: A unified web interface for analyzing security events and managing your grid.
-*   **Elastic Stack**: Powerful search backed by Elasticsearch.
-*   **Intrusion Detection**: Network-based IDS with Suricata and host-based monitoring with Elastic Fleet.
-*   **Network Metadata**: Detailed network metadata generated by Zeek or Suricata.
-*   **Full Packet Capture**: Retain and analyze raw network traffic with Suricata PCAP.
+Hunt
+![Hunt](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/56_hunt.png)

-## ⭐ Security Onion Pro
+Detections
+![Detections](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/57_detections.png)

-For organizations and enterprises requiring advanced capabilities, **Security Onion Pro** offers additional features designed for scale and efficiency:
+PCAP
+![PCAP](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/62_pcap.png)

-*   **Onion AI**: Leverage powerful AI-driven insights to accelerate your analysis and investigations.
-*   **Enterprise Features**: Enhanced tools and integrations tailored for enterprise-grade security operations.
+Grid
+![Grid](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/75_grid.png)

-For more information, visit the [Security Onion Pro](https://securityonionsolutions.com/pro) page.
+Config
+![Config](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/87_config.png)

-## ☁️ Cloud Deployment
+### Release Notes

-Security Onion is available and ready to deploy in the **AWS**, **Azure**, and **Google Cloud (GCP)** marketplaces.
+https://securityonion.net/docs/release-notes

-## 🚀 Getting Started
+### Requirements

-| Goal | Resource |
-| :--- | :--- |
-| **Download** | [Security Onion ISO](https://securityonion.net/docs/download) |
-| **Requirements** | [Hardware Guide](https://securityonion.net/docs/hardware) |
-| **Install** | [Installation Instructions](https://securityonion.net/docs/installation) |
-| **What's New** | [Release Notes](https://securityonion.net/docs/release-notes) |
+https://securityonion.net/docs/hardware

-## 📖 Documentation & Support
+### Download

-For more detailed information, please visit our [Documentation](https://docs.securityonion.net).
+https://securityonion.net/docs/download

-*   **FAQ**: [Frequently Asked Questions](https://securityonion.net/docs/faq)
-*   **Community**: [Discussions & Support](https://securityonion.net/docs/community-support)
-*   **Training**: [Official Training](https://securityonion.net/training)
+### Installation

-## 🤝 Contributing
+https://securityonion.net/docs/installation

-We welcome contributions! Please see our [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines on how to get involved.
+### FAQ

-## 🛡️ License
+https://securityonion.net/docs/faq

-Security Onion is licensed under the terms of the license found in the [LICENSE](LICENSE) file.
+### Feedback

---
-*Built with 🧅 by Security Onion Solutions.*
+https://securityonion.net/docs/community-support
@@ -4,7 +4,6 @@

 | Version | Supported          |
 | ------- | ------------------ |
-| 3.x     | :white_check_mark: |
 | 2.4.x   | :white_check_mark: |
 | 2.3.x   | :x:                |
 | 16.04.x | :x:                |
@@ -1 +1 @@
-3.1.0
+2.4.211
@@ -0,0 +1,2 @@
+elasticsearch:
+  index_settings:
@@ -1,12 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-# Per-minion Telegraf Postgres credentials. so-telegraf-cred on the manager is
-# the single writer; it mutates /opt/so/saltstack/local/pillar/telegraf/creds.sls
-# under flock. Pillar_roots order (local before default) means the populated
-# copy shadows this default on any real grid; this file exists so the pillar
-# key is always defined on fresh installs and when no minions have creds yet.
-telegraf:
-  postgres_creds: {}
@@ -17,7 +17,6 @@ base:
    - sensoroni.adv_sensoroni
    - telegraf.soc_telegraf
    - telegraf.adv_telegraf
-    - telegraf.creds
    - versionlock.soc_versionlock
    - versionlock.adv_versionlock
    - soc.license
@@ -39,9 +38,6 @@ base:
    {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
    - elasticsearch.auth
    {% endif %}
-    {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/postgres/auth.sls') %}
-    - postgres.auth
-    {% endif %}
    {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/kibana/secrets.sls') %}
    - kibana.secrets
    {% endif %}
@@ -64,8 +60,6 @@ base:
    - redis.adv_redis
    - influxdb.soc_influxdb
    - influxdb.adv_influxdb
-    - postgres.soc_postgres
-    - postgres.adv_postgres
    - elasticsearch.nodes
    - elasticsearch.soc_elasticsearch
    - elasticsearch.adv_elasticsearch
@@ -93,6 +87,8 @@ base:
    - zeek.adv_zeek
    - bpf.soc_bpf
    - bpf.adv_bpf
+    - pcap.soc_pcap
+    - pcap.adv_pcap
    - suricata.soc_suricata
    - suricata.adv_suricata
    - minions.{{ grains.id }}
@@ -103,12 +99,10 @@ base:
    - node_data.ips
    - secrets
    - healthcheck.eval
+    - elasticsearch.index_templates
    {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
    - elasticsearch.auth
    {% endif %}
-    {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/postgres/auth.sls') %}
-    - postgres.auth
-    {% endif %}
    {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/kibana/secrets.sls') %}
    - kibana.secrets
    {% endif %}
@@ -134,14 +128,14 @@ base:
    - redis.adv_redis
    - influxdb.soc_influxdb
    - influxdb.adv_influxdb
-    - postgres.soc_postgres
-    - postgres.adv_postgres
    - backup.soc_backup
    - backup.adv_backup
    - zeek.soc_zeek
    - zeek.adv_zeek
    - bpf.soc_bpf
    - bpf.adv_bpf
+    - pcap.soc_pcap
+    - pcap.adv_pcap
    - suricata.soc_suricata
    - suricata.adv_suricata
    - minions.{{ grains.id }}
@@ -152,12 +146,10 @@ base:
    - logstash.nodes
    - logstash.soc_logstash
    - logstash.adv_logstash
+    - elasticsearch.index_templates
    {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
    - elasticsearch.auth
    {% endif %}
-    {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/postgres/auth.sls') %}
-    - postgres.auth
-    {% endif %}
    {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/kibana/secrets.sls') %}
    - kibana.secrets
    {% endif %}
@@ -172,8 +164,6 @@ base:
    - redis.adv_redis
    - influxdb.soc_influxdb
    - influxdb.adv_influxdb
-    - postgres.soc_postgres
-    - postgres.adv_postgres
    - elasticsearch.nodes
    - elasticsearch.soc_elasticsearch
    - elasticsearch.adv_elasticsearch
@@ -195,6 +185,8 @@ base:
    - zeek.adv_zeek
    - bpf.soc_bpf
    - bpf.adv_bpf
+    - pcap.soc_pcap
+    - pcap.adv_pcap
    - suricata.soc_suricata
    - suricata.adv_suricata
    - minions.{{ grains.id }}
@@ -217,6 +209,8 @@ base:
    - zeek.adv_zeek
    - bpf.soc_bpf
    - bpf.adv_bpf
+    - pcap.soc_pcap
+    - pcap.adv_pcap
    - suricata.soc_suricata
    - suricata.adv_suricata
    - strelka.soc_strelka
@@ -270,12 +264,10 @@ base:
  '*_import':
    - node_data.ips
    - secrets
+    - elasticsearch.index_templates
    {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
    - elasticsearch.auth
    {% endif %}
-    {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/postgres/auth.sls') %}
-    - postgres.auth
-    {% endif %}
    {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/kibana/secrets.sls') %}
    - kibana.secrets
    {% endif %}
@@ -301,12 +293,12 @@ base:
    - redis.adv_redis
    - influxdb.soc_influxdb
    - influxdb.adv_influxdb
-    - postgres.soc_postgres
-    - postgres.adv_postgres
    - zeek.soc_zeek
    - zeek.adv_zeek
    - bpf.soc_bpf
    - bpf.adv_bpf
+    - pcap.soc_pcap
+    - pcap.adv_pcap
    - suricata.soc_suricata
    - suricata.adv_suricata
    - strelka.soc_strelka
@@ -1,14 +1,24 @@
+from os import path
 import subprocess

 def check():

+  osfam = __grains__['os_family']
  retval = 'False'

-  cmd = 'needs-restarting -r > /dev/null 2>&1'
+  if osfam == 'Debian':
+    if path.exists('/var/run/reboot-required'):
+      retval = 'True'

-  try:
-    needs_restarting = subprocess.check_call(cmd, shell=True)
-  except subprocess.CalledProcessError:
-    retval = 'True'
+  elif osfam == 'RedHat':
+    cmd = 'needs-restarting -r > /dev/null 2>&1'
+    
+    try:
+      needs_restarting = subprocess.check_call(cmd, shell=True)
+    except subprocess.CalledProcessError:
+      retval = 'True'
+
+  else:
+    retval = 'Unsupported OS: %s' % os

  return retval
@@ -29,19 +29,16 @@
    'manager',
    'nginx',
    'influxdb',
-    'postgres',
-    'postgres.auth',
    'soc',
    'kratos',
    'hydra',
    'elasticfleet',
-    'elasticfleet.manager',
-    'elasticsearch.cluster',
    'elastic-fleet-package-registry',
    'utility'
 ] %}

 {% set sensor_states = [
+    'pcap',
    'suricata',
    'healthcheck',
    'tcpreplay',
@@ -81,7 +78,7 @@
        ),
        'so-heavynode': (
            sensor_states +
-            ['elasticagent', 'elasticsearch', 'elasticsearch.cluster', 'logstash', 'redis', 'nginx']
+            ['elasticagent', 'elasticsearch', 'logstash', 'redis', 'nginx']
        ),
        'so-idh': (
            ['idh']
@@ -32,4 +32,3 @@ so_config_backup:
    - daymonth: '*'
    - month: '*'
    - dayweek: '*'
-
@@ -1,10 +1,10 @@
 backup:
  locations:
    description: List of locations to back up to the destination.
-    helpLink: backup
+    helpLink: backup.html
    global: True
  destination:
    description: Directory to store the configuration backups in.
-    helpLink: backup
+    helpLink: backup.html
    global: True
    
@@ -1,15 +1,21 @@
 {% from 'vars/globals.map.jinja' import GLOBALS %}
 {% set PCAP_BPF_STATUS = 0  %}
+{% set STENO_BPF_COMPILED = "" %}

+{% if GLOBALS.pcap_engine == "TRANSITION" %}
+{%   set PCAPBPF = ["ip and host 255.255.255.1 and port 1"] %}
+{% else %}
 {%   import_yaml 'bpf/defaults.yaml' as BPFDEFAULTS %}
 {%   set BPFMERGED = salt['pillar.get']('bpf', BPFDEFAULTS.bpf, merge=True) %}
 {%   import 'bpf/macros.jinja' as MACROS %}
 {{   MACROS.remove_comments(BPFMERGED, 'pcap') }}
 {%   set PCAPBPF = BPFMERGED.pcap %}
+{% endif %}

 {% if PCAPBPF %}
   {% set PCAP_BPF_CALC = salt['cmd.script']('salt://common/tools/sbin/so-bpf-compile', GLOBALS.sensor.interface + ' ' + PCAPBPF|join(" "),cwd='/root') %}
   {% if PCAP_BPF_CALC['retcode'] == 0 %}
      {% set PCAP_BPF_STATUS = 1 %}
+      {% set STENO_BPF_COMPILED =  ",\\\"--filter=" + PCAP_BPF_CALC['stdout'] + "\\\""  %}
   {% endif %}
 {% endif %}
@@ -3,14 +3,14 @@ bpf:
    description: List of BPF filters to apply to the PCAP engine.
    multiline: True
    forcedType: "[]string"
-    helpLink: bpf
+    helpLink: bpf.html
  suricata:
    description: List of BPF filters to apply to Suricata. This will apply to alerts and, if enabled, to metadata and PCAP logs generated by Suricata.
    multiline: True
    forcedType: "[]string"
-    helpLink: bpf
+    helpLink: bpf.html
  zeek:
    description: List of BPF filters to apply to Zeek.
    multiline: True
    forcedType: "[]string"
-    helpLink: bpf
+    helpLink: bpf.html
@@ -54,20 +54,6 @@ x509_signing_policies:
    - extendedKeyUsage: serverAuth
    - days_valid: 820
    - copypath: /etc/pki/issued_certs/
-  postgres:
-    - minions: '*'
-    - signing_private_key: /etc/pki/ca.key
-    - signing_cert: /etc/pki/ca.crt
-    - C: US
-    - ST: Utah
-    - L: Salt Lake City
-    - basicConstraints: "critical CA:false"
-    - keyUsage: "critical keyEncipherment"
-    - subjectKeyIdentifier: hash
-    - authorityKeyIdentifier: keyid,issuer:always
-    - extendedKeyUsage: serverAuth
-    - days_valid: 820
-    - copypath: /etc/pki/issued_certs/
  elasticfleet:
    - minions: '*'
    - signing_private_key: /etc/pki/ca.key
@@ -3,6 +3,8 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.

+{% from 'vars/globals.map.jinja' import GLOBALS %}
+
 include:
  - docker

@@ -16,3 +18,9 @@ trusttheca:
    - show_changes: False
    - makedirs: True

+{% if GLOBALS.os_family == 'Debian' %}
+symlinkca:
+  file.symlink:
+    - target: /etc/pki/tls/certs/intca.crt
+    - name: /etc/ssl/certs/intca.crt
+{% endif %}
@@ -0,0 +1,19 @@
+{
+  "registry-mirrors": [
+    "https://:5000"
+  ],
+  "bip": "172.17.0.1/24",
+  "default-address-pools": [
+    {
+      "base": "172.17.0.0/24",
+      "size": 24
+    }
+  ],
+  "default-ulimits": {
+      "nofile": {
+        "Name": "nofile",
+        "Soft": 1048576,
+        "Hard": 1048576
+      }
+  }
+}
@@ -20,6 +20,11 @@ kernel.printk:
  sysctl.present:
    - value: "3 4 1 3"

+# Remove variables.txt from /tmp - This is temp
+rmvariablesfile:
+  file.absent:
+    - name: /tmp/variables.txt
+
 # Add socore Group
 socoregroup:
  group.present:
@@ -144,6 +149,28 @@ common_sbin_jinja:
      - so-import-pcap
 {% endif %}

+{% if GLOBALS.role == 'so-heavynode' %}
+remove_so-pcap-import_heavynode:
+  file.absent:
+    - name: /usr/sbin/so-pcap-import
+
+remove_so-import-pcap_heavynode:
+  file.absent:
+    - name: /usr/sbin/so-import-pcap
+{% endif %}
+
+{% if not GLOBALS.is_manager%}
+# prior to 2.4.50 these scripts were in common/tools/sbin on the manager because of soup and distributed to non managers
+# these two states remove the scripts from non manager nodes
+remove_soup:
+  file.absent:
+    - name: /usr/sbin/soup
+
+remove_so-firewall:
+  file.absent:
+    - name: /usr/sbin/so-firewall
+{% endif %}
+
 so-status_script:
  file.managed:
    - name: /usr/sbin/so-status
@@ -1,5 +1,52 @@
 # we cannot import GLOBALS from vars/globals.map.jinja in this state since it is called in setup.virt.init
 # since it is early in setup of a new VM, the pillars imported in GLOBALS are not yet defined
+{% if grains.os_family == 'Debian' %}
+commonpkgs:
+  pkg.installed:
+    - skip_suggestions: True
+    - pkgs:
+      - apache2-utils
+      - wget
+      - ntpdate
+      - jq
+      - curl
+      - ca-certificates
+      - software-properties-common
+      - apt-transport-https
+      - openssl
+      - netcat-openbsd
+      - sqlite3
+      - libssl-dev
+      - procps
+      - python3-dateutil
+      - python3-docker
+      - python3-packaging
+      - python3-lxml
+      - git
+      - rsync
+      - vim
+      - tar
+      - unzip
+      - bc
+      {% if grains.oscodename != 'focal' %}
+      - python3-rich
+      {% endif %}
+
+{%     if grains.oscodename == 'focal' %}
+# since Ubuntu requires and internet connection we can use pip to install modules
+python3-pip:
+  pkg.installed
+
+python-rich:
+  pip.installed:
+    - name: rich
+    - target: /usr/local/lib/python3.8/dist-packages/
+    - require:
+      - pkg: python3-pip
+{%     endif %}
+{% endif %}
+
+{% if grains.os_family == 'RedHat' %}

 remove_mariadb:
  pkg.removed:
@@ -37,3 +84,5 @@ commonpkgs:
      - unzip
      - wget
      - yum-utils
+
+{% endif %}
@@ -3,6 +3,9 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.

+{% set soversion = salt['cp.get_file_str']('/etc/soversion') %}
+{% if '2.4' in soversion or soversion.startswith('3.') %}
+
 {%   import_yaml '/opt/so/saltstack/local/pillar/global/soc_global.sls' as SOC_GLOBAL %}
 {%   if SOC_GLOBAL.global.airgap %}
 {%     set UPDATE_DIR='/tmp/soagupdate/SecurityOnion' %}
@@ -11,6 +14,14 @@
 {%   endif %}
 {%   set SOVERSION = salt['file.read']('/etc/soversion').strip() %}

+remove_common_soup:
+  file.absent:
+    - name: /opt/so/saltstack/default/salt/common/tools/sbin/soup
+
+remove_common_so-firewall:
+  file.absent:
+    - name: /opt/so/saltstack/default/salt/common/tools/sbin/so-firewall
+
 # This section is used to put the scripts in place in the Salt file system
 # in case a state run tries to overwrite what we do in the next section.
 copy_so-common_common_tools_sbin:
@@ -110,3 +121,23 @@ copy_bootstrap-salt_sbin:
    - source: {{UPDATE_DIR}}/salt/salt/scripts/bootstrap-salt.sh
    - force: True
    - preserve: True
+
+{# this is added in 2.4.120 to remove salt repo files pointing to saltproject.io to accomodate the move to broadcom and new bootstrap-salt script #}
+{%   if salt['pkg.version_cmp'](SOVERSION, '2.4.120') == -1 %}
+{%     set saltrepofile = '/etc/yum.repos.d/salt.repo' %}
+{%     if grains.os_family == 'Debian' %}
+{%       set saltrepofile = '/etc/apt/sources.list.d/salt.list' %}
+{%     endif %}
+remove_saltproject_io_repo_manager:
+  file.absent:
+    - name: {{ saltrepofile }}
+{%   endif %}
+
+{% else %}
+fix_23_soup_sbin:
+  cmd.run:
+    - name: curl -s -f -o /usr/sbin/soup https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.3/main/salt/common/tools/sbin/soup
+fix_23_soup_salt:
+  cmd.run:
+    - name: curl -s -f -o /opt/so/saltstack/defalt/salt/common/tools/sbin/soup https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.3/main/salt/common/tools/sbin/soup
+{% endif %}
@@ -16,7 +16,7 @@

 if [ "$#" -lt 2 ]; then
  cat 1>&2 <<EOF
-$0 compiles a BPF expression to be passed to PCAP to apply a socket filter.
+$0 compiles a BPF expression to be passed to stenotype to apply a socket filter.
 Its first argument is the interface (link type is required) and all other arguments
 are passed to TCPDump.

@@ -333,8 +333,8 @@ get_elastic_agent_vars() {

 	if [ -f "$defaultsfile" ]; then
 		ELASTIC_AGENT_TARBALL_VERSION=$(egrep " +version: " $defaultsfile | awk -F: '{print $2}' | tr -d '[:space:]')
-		ELASTIC_AGENT_URL="https://repo.securityonion.net/file/so-repo/prod/3/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz"
-		ELASTIC_AGENT_MD5_URL="https://repo.securityonion.net/file/so-repo/prod/3/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5"
+		ELASTIC_AGENT_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz"
+		ELASTIC_AGENT_MD5_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5"
 		ELASTIC_AGENT_FILE="/nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz"
 		ELASTIC_AGENT_MD5="/nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5"
 		ELASTIC_AGENT_EXPANSION_DIR=/nsm/elastic-fleet/artifacts/beats/elastic-agent
@@ -349,16 +349,21 @@ get_random_value() {
 }

 gpg_rpm_import() {
-	if [[ "$WHATWOULDYOUSAYYAHDOHERE" == "setup" ]]; then
-		local RPMKEYSLOC="../salt/repo/client/files/$OS/keys"
-	else
-		local RPMKEYSLOC="$UPDATE_DIR/salt/repo/client/files/$OS/keys"
+	if [[ $is_oracle ]]; then
+	    if [[ "$WHATWOULDYOUSAYYAHDOHERE" == "setup" ]]; then
+	        local RPMKEYSLOC="../salt/repo/client/files/$OS/keys"
+	    else
+	        local RPMKEYSLOC="$UPDATE_DIR/salt/repo/client/files/$OS/keys"
+	    fi
+		RPMKEYS=('RPM-GPG-KEY-oracle' 'RPM-GPG-KEY-EPEL-9' 'SALT-PROJECT-GPG-PUBKEY-2023.pub' 'docker.pub' 'securityonion.pub')
+		for RPMKEY in "${RPMKEYS[@]}"; do
+    	        rpm --import $RPMKEYSLOC/$RPMKEY
+	        echo "Imported $RPMKEY"
+	    done
+	elif [[ $is_rpm ]]; then
+		echo "Importing the security onion GPG key"
+		rpm --import ../salt/repo/client/files/oracle/keys/securityonion.pub
 	fi
-	RPMKEYS=('RPM-GPG-KEY-oracle' 'RPM-GPG-KEY-EPEL-9' 'SALT-PROJECT-GPG-PUBKEY-2023.pub' 'docker.pub' 'securityonion.pub')
-	for RPMKEY in "${RPMKEYS[@]}"; do
-		rpm --import $RPMKEYSLOC/$RPMKEY
-		echo "Imported $RPMKEY"
-	done
 }

 header() {
@@ -545,22 +550,6 @@ retry() {
        return $exitcode
 }

-rollover_index() {
-  idx=$1
-  exists=$(so-elasticsearch-query $idx -o /dev/null -w "%{http_code}")
-  if [[ $exists -eq 200 ]]; then
-    rollover=$(so-elasticsearch-query $idx/_rollover -o /dev/null -w "%{http_code}" -XPOST)
-
-    if [[ $rollover -eq 200 ]]; then
-      echo "Successfully triggered rollover for $idx..."
-    else
-      echo "Could not trigger rollover for $idx..."
-    fi
-  else
-    echo "Could not find index $idx..."
-  fi
-}
-
 run_check_net_err() {
 	local cmd=$1
 	local err_msg=${2:-"Unknown error occured, please check /root/$WHATWOULDYOUSAYYAHDOHERE.log for details."} # Really need to rename that variable
@@ -626,19 +615,69 @@ salt_minion_count() {
 }

 set_os() {
-	if [ -f /etc/redhat-release ] && grep -q "Red Hat Enterprise Linux release 9" /etc/redhat-release && [ -f /etc/oracle-release ]; then
-		OS=oracle
-		OSVER=9
-		is_oracle=true
-		is_rpm=true
+	if [ -f /etc/redhat-release ]; then
+		if grep -q "Rocky Linux release 9" /etc/redhat-release; then
+			OS=rocky
+			OSVER=9
+			is_rocky=true
+			is_rpm=true
+		elif grep -q "CentOS Stream release 9" /etc/redhat-release; then
+			OS=centos
+			OSVER=9
+			is_centos=true
+			is_rpm=true
+		elif grep -q "AlmaLinux release 9" /etc/redhat-release; then
+			OS=alma
+			OSVER=9
+			is_alma=true
+			is_rpm=true
+		elif grep -q "Red Hat Enterprise Linux release 9" /etc/redhat-release; then
+			if [ -f /etc/oracle-release ]; then
+				OS=oracle
+				OSVER=9
+				is_oracle=true
+				is_rpm=true
+			else
+				OS=rhel
+				OSVER=9
+				is_rhel=true
+				is_rpm=true
+			fi
+		fi
+		cron_service_name="crond"
+	elif [ -f /etc/os-release ]; then
+		if grep -q "UBUNTU_CODENAME=focal" /etc/os-release; then
+			OSVER=focal
+			UBVER=20.04
+			OS=ubuntu
+			is_ubuntu=true
+			is_deb=true
+		elif grep -q "UBUNTU_CODENAME=jammy" /etc/os-release; then
+			OSVER=jammy
+			UBVER=22.04
+			OS=ubuntu
+			is_ubuntu=true
+			is_deb=true
+		elif grep -q "VERSION_CODENAME=bookworm" /etc/os-release; then
+			OSVER=bookworm
+			DEBVER=12
+			is_debian=true
+			OS=debian
+			is_deb=true
+		fi
+		cron_service_name="cron"
 	fi
-	cron_service_name="crond"
 }

 set_minionid() {
 	MINIONID=$(lookup_grain id)
 }

+set_palette() {
+	if [[ $is_deb ]]; then
+	    update-alternatives --set newt-palette /etc/newt/palette.original
+    fi
+}

 set_version() {
 	CURRENTVERSION=0.0.0
@@ -31,8 +31,8 @@ container_list() {
      "so-hydra"
      "so-nginx"
      "so-pcaptools"
-      "so-postgres"
      "so-soc"
+      "so-steno"
      "so-suricata"
      "so-telegraf"
      "so-zeek"
@@ -56,9 +56,9 @@ container_list() {
      "so-logstash"
      "so-nginx"
      "so-pcaptools"
-      "so-postgres"
      "so-redis"
      "so-soc"
+      "so-steno"
      "so-strelka-backend"
      "so-strelka-manager"
      "so-suricata"
@@ -71,6 +71,7 @@ container_list() {
      "so-logstash"
      "so-nginx"
      "so-redis"
+      "so-steno"
      "so-suricata"
      "so-soc"
      "so-telegraf"
@@ -164,8 +165,8 @@ update_docker_containers() {
    # Pull down the trusted docker image
    run_check_net_err \
    "docker pull $CONTAINER_REGISTRY/$IMAGEREPO/$image" \
-    "Could not pull $image, please ensure connectivity to $CONTAINER_REGISTRY" >> "$LOG_FILE" 2>&1
-
+    "Could not pull $image, please ensure connectivity to $CONTAINER_REGISTRY" >> "$LOG_FILE" 2>&1 
+    
    # Get signature
    run_check_net_err \
    "curl --retry 5 --retry-delay 60 -A '$CURLTYPE/$CURRENTVERSION/$OS/$(uname -r)' $sig_url --output $SIGNPATH/$image.sig" \
@@ -188,27 +189,8 @@ update_docker_containers() {
        if [ -z "$HOSTNAME" ]; then
          HOSTNAME=$(hostname)
        fi
-        docker tag $CONTAINER_REGISTRY/$IMAGEREPO/$image $HOSTNAME:5000/$IMAGEREPO/$image >> "$LOG_FILE" 2>&1 || {
-          echo "Unable to tag $image" >> "$LOG_FILE" 2>&1
-          exit 1
-        }
-        # Push to the embedded registry via a registry-to-registry copy. Avoids
-        # `docker push`, which on Docker 29.x with the containerd image store
-        # represents freshly-pulled images as an index whose layer content
-        # isn't reachable through the push path. The local `docker tag` above
-        # is preserved so so-image-pull's `:5000` existence check still works.
-        # Pin to the digest already gpg-verified above so we copy exactly the
-        # bytes we approved.
-        local VERIFIED_REF
-        VERIFIED_REF=$(echo "$DOCKERINSPECT" | jq -r ".[0].RepoDigests[] | select(. | contains(\"$CONTAINER_REGISTRY\"))" | head -n 1)
-        if [ -z "$VERIFIED_REF" ] || [ "$VERIFIED_REF" = "null" ]; then
-          echo "Unable to determine verified digest for $image" >> "$LOG_FILE" 2>&1
-          exit 1
-        fi
-        docker buildx imagetools create --tag $HOSTNAME:5000/$IMAGEREPO/$image "$VERIFIED_REF" >> "$LOG_FILE" 2>&1 || {
-          echo "Unable to copy $image to embedded registry" >> "$LOG_FILE" 2>&1
-          exit 1
-        }
+        docker tag $CONTAINER_REGISTRY/$IMAGEREPO/$image $HOSTNAME:5000/$IMAGEREPO/$image >> "$LOG_FILE" 2>&1 
+        docker push $HOSTNAME:5000/$IMAGEREPO/$image >> "$LOG_FILE" 2>&1 
      fi
    else
      echo "There is a problem downloading the $image image. Details: " >> "$LOG_FILE" 2>&1 
@@ -131,7 +131,6 @@ if [[ $EXCLUDE_STARTUP_ERRORS == 'Y' ]]; then
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|not configured for GeoIP"     # SO does not bundle the maxminddb with Zeek
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|HTTP 404: Not Found"          # Salt loops until Kratos returns 200, during startup Kratos may not be ready
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Cancelling deferred write event maybeFenceReplicas because the event queue is now closed" # Kafka controller log during shutdown/restart
-    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Redis may have been restarted" # Redis likely restarted by salt
 fi

 if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then
@@ -180,6 +179,7 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|salt-minion-check"            # bug in early 2.4 place Jinja script in non-jinja salt dir causing cron output errors
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|monitoring.metrics"           # known issue with elastic agent casting the field incorrectly if an integer value shows up before a float
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|repodownload.conf"            # known issue with reposync on pre-2.4.20 
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|missing versions record"      # stenographer corrupt index
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|soc.field."                   # known ingest type collisions issue with earlier versions of SO
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|error parsing signature"      # Malformed Suricata rule, from upstream provider
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|sticky buffer has no matches" # Non-critical Suricata error
@@ -227,7 +227,7 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|from NIC checksum offloading" # zeek reporter.log
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|marked for removal"           # docker container getting recycled
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|tcp 127.0.0.1:6791: bind: address already in use" # so-elastic-fleet agent restarting. Seen starting w/ 8.18.8 https://github.com/elastic/kibana/issues/201459
-    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|TransformTask\] \[logs-(tychon|aws_billing|microsoft_defender_endpoint|armis|o365_metrics|microsoft_sentinel|snyk|cyera|island_browser).*user so_kibana lacks the required permissions \[(logs|metrics)-\1" # Known issue with integrations starting transform jobs that are explicitly not allowed to start as a system user. This error should not be seen on fresh ES 9.3.3 installs or after SO 3.1.0 with soups addition of check_transform_health_and_reauthorize()
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|TransformTask\] \[logs-(tychon|aws_billing|microsoft_defender_endpoint).*user so_kibana lacks the required permissions \[logs-\1" # Known issue with 3 integrations using kibana_system role vs creating unique api creds with proper permissions.
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|manifest unknown"             # appears in so-dockerregistry log for so-tcpreplay following docker upgrade to 29.2.1-1
 fi

@@ -55,22 +55,19 @@ if [ $SKIP -ne 1 ]; then
 fi

 delete_pcap() {
-  PCAP_DATA="/nsm/suripcap/"
-  [ -d $PCAP_DATA ] && rm -rf $PCAP_DATA/*
+  PCAP_DATA="/nsm/pcap/"
+  [ -d $PCAP_DATA ] && so-pcap-stop && rm -rf $PCAP_DATA/* && so-pcap-start
 }
 delete_suricata() {
  SURI_LOG="/nsm/suricata/"
-  [ -d $SURI_LOG ] && rm -rf $SURI_LOG/*
+  [ -d $SURI_LOG ] && so-suricata-stop && rm -rf $SURI_LOG/* && so-suricata-start
 }
 delete_zeek() {
  ZEEK_LOG="/nsm/zeek/logs/"
  [ -d $ZEEK_LOG ] && so-zeek-stop && rm -rf $ZEEK_LOG/* && so-zeek-start
 }

-so-suricata-stop
 delete_pcap
 delete_suricata
 delete_zeek
-so-suricata-start
-

@@ -23,6 +23,7 @@ if [ $# -ge 1 ]; then
        fi

        case $1 in
+                "steno") docker stop so-steno && docker rm so-steno && salt-call state.apply pcap queue=True;;
                "elastic-fleet") docker stop so-elastic-fleet && docker rm so-elastic-fleet && salt-call state.apply elasticfleet queue=True;;
                *)  docker stop so-$1 ; docker rm so-$1 ; salt-call state.apply $1 queue=True;;
        esac
@@ -72,7 +72,7 @@ clean() {
 		done
 	fi

-	## Clean up extracted pcaps
+	## Clean up extracted pcaps from Steno
 	PCAPS='/nsm/pcapout'
 	OLDEST_PCAP=$(find $PCAPS -type f -printf '%T+ %p\n' | sort -n | head -n 1)
 	if [ -z "$OLDEST_PCAP" -o "$OLDEST_PCAP" == ".." -o "$OLDEST_PCAP" == "." ]; then
@@ -23,6 +23,7 @@ if [ $# -ge 1 ]; then

 	case $1 in
   		"all") salt-call state.highstate queue=True;;
+   		"steno") if docker ps | grep -q so-$1; then printf "\n$1 is already running!\n\n"; else docker rm so-$1 >/dev/null 2>&1 ; salt-call state.apply pcap queue=True; fi ;; 
   		"elastic-fleet") if docker ps | grep -q so-$1; then printf "\n$1 is already running!\n\n"; else docker rm so-$1 >/dev/null 2>&1 ; salt-call state.apply elasticfleet queue=True; fi ;; 
   		*) if docker ps | grep -E -q '^so-$1$'; then printf "\n$1 is already running\n\n"; else docker rm so-$1 >/dev/null 2>&1 ; salt-call state.apply $1 queue=True; fi ;; 
 	esac
@@ -9,7 +9,7 @@

 . /usr/sbin/so-common

-software_raid=("SOSMN" "SOSMN-DE02" "SOSSNNV" "SOSSNNV-DE02" "SOS10k-DE02" "SOS10KNV" "SOS10KNV-DE02" "SOS10KNV-DE02" "SOS2000-DE02" "SOS-GOFAST-LT-DE02" "SOS-GOFAST-MD-DE02" "SOS-GOFAST-HV-DE02" "HVGUEST")
+software_raid=("SOSMN" "SOSMN-DE02" "SOSSNNV" "SOSSNNV-DE02" "SOS10k-DE02" "SOS10KNV" "SOS10KNV-DE02" "SOS10KNV-DE02" "SOS2000-DE02" "SOS-GOFAST-LT-DE02" "SOS-GOFAST-MD-DE02" "SOS-GOFAST-HV-DE02")
 hardware_raid=("SOS1000" "SOS1000F" "SOSSN7200" "SOS5000" "SOS4000")

 {%- if salt['grains.get']('sosmodel', '') %}
@@ -87,11 +87,6 @@ check_boss_raid() {
 }

 check_software_raid() {
-  if [[ ! -f /proc/mdstat ]]; then
-    SWRAID=0
-    return
-  fi
-
  SWRC=$(grep "_" /proc/mdstat)
  if [[ -n $SWRC ]]; then
      # RAID is failed in some way
@@ -112,9 +107,7 @@ if [[ "$is_hwraid" == "true" ]]; then
 fi
 if [[ "$is_softwareraid" == "true" ]]; then
 	check_software_raid
-  if [ "$model" != "HVGUEST" ]; then
-    check_boss_raid
-  fi
+  check_boss_raid
 fi

 sum=$(($SWRAID + $BOSSRAID + $HWRAID))
@@ -0,0 +1,34 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+so-curator:
+  docker_container.absent:
+    - force: True
+
+so-curator_so-status.disabled:
+  file.line:
+    - name: /opt/so/conf/so-status/so-status.conf
+    - match: ^so-curator$
+    - mode: delete
+
+so-curator-cluster-close:
+  cron.absent:
+    - identifier: so-curator-cluster-close
+
+so-curator-cluster-delete:
+  cron.absent:
+    - identifier: so-curator-cluster-delete
+
+delete_curator_configuration:
+  file.absent:
+    - name: /opt/so/conf/curator
+    - recurse: True
+
+{% set files = salt.file.find(path='/usr/sbin', name='so-curator*') %}
+{% if files|length > 0 %}
+delete_curator_scripts:
+  file.absent:
+    - names: {{files|yaml}}
+{% endif %}
@@ -1,10 +1,6 @@
 docker:
  range: '172.17.1.0/24'
  gateway: '172.17.1.1'
-  ulimits:
-    - name: nofile
-      soft: 1048576
-      hard: 1048576
  containers:
    'so-dockerregistry':
      final_octet: 20
@@ -13,7 +9,6 @@ docker:
      custom_bind_mounts: []
      extra_hosts: []
      extra_env: []
-      ulimits: []
    'so-elastic-fleet':
      final_octet: 21
      port_bindings:
@@ -21,7 +16,6 @@ docker:
      custom_bind_mounts: []
      extra_hosts: []
      extra_env: []
-      ulimits: []
    'so-elasticsearch':
      final_octet: 22
      port_bindings:
@@ -30,16 +24,6 @@ docker:
      custom_bind_mounts: []
      extra_hosts: []
      extra_env: []
-      ulimits:
-        - name: memlock
-          soft: -1
-          hard: -1
-        - name: nofile
-          soft: 65536
-          hard: 65536
-        - name: nproc
-          soft: 4096
-          hard: 4096
    'so-influxdb':
      final_octet: 26
      port_bindings:
@@ -47,7 +31,6 @@ docker:
      custom_bind_mounts: []
      extra_hosts: []
      extra_env: []
-      ulimits: []
    'so-kibana':
      final_octet: 27
      port_bindings:
@@ -55,7 +38,6 @@ docker:
      custom_bind_mounts: []
      extra_hosts: []
      extra_env: []
-      ulimits: []
    'so-kratos':
      final_octet: 28
      port_bindings:
@@ -64,7 +46,6 @@ docker:
      custom_bind_mounts: []
      extra_hosts: []
      extra_env: []
-      ulimits: []
    'so-hydra':
      final_octet: 30
      port_bindings:
@@ -73,7 +54,6 @@ docker:
      custom_bind_mounts: []
      extra_hosts: []
      extra_env: []
-      ulimits: []
    'so-logstash':
      final_octet: 29
      port_bindings:
@@ -90,7 +70,6 @@ docker:
      custom_bind_mounts: []
      extra_hosts: []
      extra_env: []
-      ulimits: []
    'so-nginx':
      final_octet: 31
      port_bindings:
@@ -102,7 +81,6 @@ docker:
      custom_bind_mounts: []
      extra_hosts: []
      extra_env: []
-      ulimits: []
    'so-nginx-fleet-node':
      final_octet: 31
      port_bindings:
@@ -110,7 +88,6 @@ docker:
      custom_bind_mounts: []
      extra_hosts: []
      extra_env: []
-      ulimits: []
    'so-redis':
      final_octet: 33
      port_bindings:
@@ -119,13 +96,11 @@ docker:
      custom_bind_mounts: []
      extra_hosts: []
      extra_env: []
-      ulimits: []
    'so-sensoroni':
      final_octet: 99
      custom_bind_mounts: []
      extra_hosts: []
      extra_env: []
-      ulimits: []
    'so-soc':
      final_octet: 34
      port_bindings:
@@ -133,19 +108,16 @@ docker:
      custom_bind_mounts: []
      extra_hosts: []
      extra_env: []
-      ulimits: []
    'so-strelka-backend':
      final_octet: 36
      custom_bind_mounts: []
      extra_hosts: []
      extra_env: []
-      ulimits: []
    'so-strelka-filestream':
      final_octet: 37
      custom_bind_mounts: []
      extra_hosts: []
      extra_env: []
-      ulimits: []
    'so-strelka-frontend':
      final_octet: 38
      port_bindings:
@@ -153,13 +125,11 @@ docker:
      custom_bind_mounts: []
      extra_hosts: []
      extra_env: []
-      ulimits: []
    'so-strelka-manager':
      final_octet: 39
      custom_bind_mounts: []
      extra_hosts: []
      extra_env: []
-      ulimits: []
    'so-strelka-gatekeeper':
      final_octet: 40
      port_bindings:
@@ -167,7 +137,6 @@ docker:
      custom_bind_mounts: []
      extra_hosts: []
      extra_env: []
-      ulimits: []
    'so-strelka-coordinator':
      final_octet: 41
      port_bindings:
@@ -175,13 +144,11 @@ docker:
      custom_bind_mounts: []
      extra_hosts: []
      extra_env: []
-      ulimits: []
    'so-elastalert':
      final_octet: 42
      custom_bind_mounts: []
      extra_hosts: []
      extra_env: []
-      ulimits: []
    'so-elastic-fleet-package-registry':
      final_octet: 44
      port_bindings:
@@ -189,13 +156,11 @@ docker:
      custom_bind_mounts: []
      extra_hosts: []
      extra_env: []
-      ulimits: []
    'so-idh':
      final_octet: 45
      custom_bind_mounts: []
      extra_hosts: []
      extra_env: []
-      ulimits: []
    'so-elastic-agent':
      final_octet: 46
      port_bindings:
@@ -204,28 +169,28 @@ docker:
      custom_bind_mounts: []
      extra_hosts: []
      extra_env: []
-      ulimits: []
    'so-telegraf':
      final_octet: 99
      custom_bind_mounts: []
      extra_hosts: []
      extra_env: []
-      ulimits: []
+    'so-steno':
+      final_octet: 99
+      custom_bind_mounts: []
+      extra_hosts: []
+      extra_env: []
    'so-suricata':
      final_octet: 99
      custom_bind_mounts: []
      extra_hosts: []
      extra_env: []
-      ulimits: []
+      ulimits:
+        - memlock=524288000
    'so-zeek':
      final_octet: 99
      custom_bind_mounts: []
      extra_hosts: []
      extra_env: []
-      ulimits:
-        - name: core
-          soft: 0
-          hard: 0
    'so-kafka':
      final_octet: 88
      port_bindings:
@@ -236,12 +201,3 @@ docker:
      custom_bind_mounts: []
      extra_hosts: []
      extra_env: []
-      ulimits: []
-    'so-postgres':
-      final_octet: 47
-      port_bindings:
-        - 0.0.0.0:5432:5432
-      custom_bind_mounts: []
-      extra_hosts: []
-      extra_env: []
-      ulimits: []
@@ -1,8 +1,8 @@
 {% import_yaml 'docker/defaults.yaml' as DOCKERDEFAULTS %}
-{% set DOCKERMERGED = salt['pillar.get']('docker', DOCKERDEFAULTS.docker, merge=True) %}
-{% set RANGESPLIT = DOCKERMERGED.range.split('.') %}
+{% set DOCKER = salt['pillar.get']('docker', DOCKERDEFAULTS.docker, merge=True) %}
+{% set RANGESPLIT = DOCKER.range.split('.') %}
 {% set FIRSTTHREE = RANGESPLIT[0] ~ '.' ~ RANGESPLIT[1] ~ '.' ~ RANGESPLIT[2] ~ '.' %}

-{% for container, vals in DOCKERMERGED.containers.items() %}
-{%   do DOCKERMERGED.containers[container].update({'ip': FIRSTTHREE ~ DOCKERMERGED.containers[container].final_octet}) %}
+{% for container, vals in DOCKER.containers.items() %}
+{%   do DOCKER.containers[container].update({'ip': FIRSTTHREE ~ DOCKER.containers[container].final_octet}) %}
 {% endfor %}
@@ -1,24 +0,0 @@
-{% from 'docker/docker.map.jinja' import DOCKERMERGED -%}
-{
-  "registry-mirrors": [
-    "https://:5000"
-  ],
-  "bip": "172.17.0.1/24",
-  "default-address-pools": [
-    {
-      "base": "172.17.0.0/24",
-      "size": 24
-    }
-  ]
-{%- if DOCKERMERGED.ulimits %},
-  "default-ulimits": {
-{%-   for ULIMIT in DOCKERMERGED.ulimits %}
-      "{{ ULIMIT.name }}": {
-        "Name": "{{ ULIMIT.name }}",
-        "Soft": {{ ULIMIT.soft }},
-        "Hard": {{ ULIMIT.hard }}
-      }{{ "," if not loop.last else "" }}
-{%-   endfor %}
-  }
-{%- endif %}
-}
@@ -3,7 +3,7 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.

-{% from 'docker/docker.map.jinja' import DOCKERMERGED %}
+{% from 'docker/docker.map.jinja' import DOCKER %}
 {% from 'vars/globals.map.jinja' import GLOBALS %}

 # docker service requires the ca.crt
@@ -15,6 +15,39 @@ dockergroup:
    - name: docker
    - gid: 920

+{% if GLOBALS.os_family == 'Debian' %}
+{%    if grains.oscodename == 'bookworm' %}
+dockerheldpackages:
+  pkg.installed:
+    - pkgs:
+      - containerd.io: 2.2.1-1~debian.12~bookworm
+      - docker-ce: 5:29.2.1-1~debian.12~bookworm
+      - docker-ce-cli: 5:29.2.1-1~debian.12~bookworm
+      - docker-ce-rootless-extras: 5:29.2.1-1~debian.12~bookworm
+    - hold: True
+    - update_holds: True
+{%    elif grains.oscodename == 'jammy' %}
+dockerheldpackages:
+  pkg.installed:
+    - pkgs:
+      - containerd.io: 2.2.1-1~ubuntu.22.04~jammy
+      - docker-ce: 5:29.2.1-1~ubuntu.22.04~jammy
+      - docker-ce-cli: 5:29.2.1-1~ubuntu.22.04~jammy
+      - docker-ce-rootless-extras: 5:29.2.1-1~ubuntu.22.04~jammy
+    - hold: True
+    - update_holds: True
+{%    else %}
+dockerheldpackages:
+  pkg.installed:
+    - pkgs:
+      - containerd.io: 1.7.21-1
+      - docker-ce: 5:27.2.0-1~ubuntu.20.04~focal
+      - docker-ce-cli: 5:27.2.0-1~ubuntu.20.04~focal
+      - docker-ce-rootless-extras: 5:27.2.0-1~ubuntu.20.04~focal
+    - hold: True
+    - update_holds: True
+{%   endif %}
+{% else %}
 dockerheldpackages:
  pkg.installed:
    - pkgs:
@@ -24,6 +57,7 @@ dockerheldpackages:
      - docker-ce-rootless-extras: 29.2.1-1.el9
    - hold: True
    - update_holds: True
+{% endif %}

 #disable docker from managing iptables
 iptables_disabled:
@@ -41,9 +75,10 @@ dockeretc:
  file.directory:
    - name: /etc/docker

+# Manager daemon.json
 docker_daemon:
  file.managed:
-    - source: salt://docker/files/daemon.json.jinja
+    - source: salt://common/files/daemon.json
    - name: /etc/docker/daemon.json
    - template: jinja 

@@ -74,8 +109,8 @@ dockerreserveports:
 sos_docker_net:
  docker_network.present:
    - name: sobridge
-    - subnet: {{ DOCKERMERGED.range }}
-    - gateway: {{ DOCKERMERGED.gateway }}
+    - subnet: {{ DOCKER.range }}
+    - gateway: {{ DOCKER.gateway }}
    - options:
        com.docker.network.bridge.name: 'sobridge'
        com.docker.network.driver.mtu: '1500'
@@ -1,82 +1,44 @@
 docker:
  gateway:
    description: Gateway for the default docker interface.
-    helpLink: docker
+    helpLink: docker.html
    advanced: True
  range:
    description: Default docker IP range for containers.
-    helpLink: docker
-    advanced: True
-  ulimits:
-    description: |
-      Default ulimit settings applied to all containers via the Docker daemon. Each entry specifies a resource name (e.g. nofile, memlock, core, nproc) with soft and hard limits. Individual container ulimits override these defaults. Valid resource names include: cpu, fsize, data, stack, core, rss, nproc, nofile, memlock, as, locks, sigpending, msgqueue, nice, rtprio, rttime.
-    forcedType: "[]{}"
-    syntax: json
-    advanced: True
    helpLink: docker.html
-    uiElements:
-    - field: name
-      label: Resource Name
-      required: True
-      regex: ^(cpu|fsize|data|stack|core|rss|nproc|nofile|memlock|as|locks|sigpending|msgqueue|nice|rtprio|rttime)$
-      regexFailureMessage: You must enter a valid ulimit name (cpu, fsize, data, stack, core, rss, nproc, nofile, memlock, as, locks, sigpending, msgqueue, nice, rtprio, rttime).
-    - field: soft
-      label: Soft Limit
-      forcedType: int
-    - field: hard
-      label: Hard Limit
-      forcedType: int
+    advanced: True
  containers:
    so-dockerregistry: &dockerOptions
      final_octet:
        description: Last octet of the container IP address.
-        helpLink: docker
+        helpLink: docker.html
        readonly: True
        advanced: True
        global: True
      port_bindings:
        description: List of port bindings for the container.
-        helpLink: docker
+        helpLink: docker.html
        advanced: True
        multiline: True
        forcedType: "[]string"
      custom_bind_mounts:
        description: List of custom local volume bindings.
        advanced: True
-        helpLink: docker
+        helpLink: docker.html
        multiline: True
        forcedType: "[]string"
      extra_hosts:
        description: List of additional host entries for the container.
        advanced: True
-        helpLink: docker
+        helpLink: docker.html
        multiline: True
        forcedType: "[]string"
      extra_env:
        description: List of additional ENV entries for the container.
        advanced: True
-        helpLink: docker
+        helpLink: docker.html
        multiline: True
        forcedType: "[]string"
-      ulimits:
-        description: |
-          Ulimit settings for the container. Each entry specifies a resource name (e.g. nofile, memlock, core, nproc) with optional soft and hard limits. Valid resource names include: cpu, fsize, data, stack, core, rss, nproc, nofile, memlock, as, locks, sigpending, msgqueue, nice, rtprio, rttime.
-        advanced: True
-        helpLink: docker.html
-        forcedType: "[]{}"
-        syntax: json
-        uiElements:
-        - field: name
-          label: Resource Name
-          required: True
-          regex: ^(cpu|fsize|data|stack|core|rss|nproc|nofile|memlock|as|locks|sigpending|msgqueue|nice|rtprio|rttime)$
-          regexFailureMessage: You must enter a valid ulimit name (cpu, fsize, data, stack, core, rss, nproc, nofile, memlock, as, locks, sigpending, msgqueue, nice, rtprio, rttime).
-        - field: soft
-          label: Soft Limit
-          forcedType: int
-        - field: hard
-          label: Hard Limit
-          forcedType: int
    so-elastic-fleet: *dockerOptions
    so-elasticsearch: *dockerOptions
    so-influxdb: *dockerOptions
@@ -100,6 +62,43 @@ docker:
    so-idh: *dockerOptions
    so-elastic-agent: *dockerOptions
    so-telegraf: *dockerOptions
-    so-suricata: *dockerOptions
+    so-steno: *dockerOptions
+    so-suricata:
+      final_octet:
+        description: Last octet of the container IP address.
+        helpLink: docker.html
+        readonly: True
+        advanced: True
+        global: True
+      port_bindings:
+        description: List of port bindings for the container.
+        helpLink: docker.html
+        advanced: True
+        multiline: True
+        forcedType: "[]string"
+      custom_bind_mounts:
+        description: List of custom local volume bindings.
+        advanced: True
+        helpLink: docker.html
+        multiline: True
+        forcedType: "[]string"
+      extra_hosts:
+        description: List of additional host entries for the container.
+        advanced: True
+        helpLink: docker.html
+        multiline: True
+        forcedType: "[]string"
+      extra_env:
+        description: List of additional ENV entries for the container.
+        advanced: True
+        helpLink: docker.html
+        multiline: True
+        forcedType: "[]string"
+      ulimits:
+        description: Ulimits for the container, in bytes.
+        advanced: True
+        helpLink: docker.html
+        multiline: True
+        forcedType: "[]string"
    so-zeek: *dockerOptions
    so-kafka: *dockerOptions
@@ -6,7 +6,7 @@
 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
-{%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
+{%   from 'docker/docker.map.jinja' import DOCKER %}

 include:
  - elastalert.config
@@ -24,7 +24,7 @@ so-elastalert:
    - user: so-elastalert
    - networks:
      - sobridge:
-        - ipv4_address: {{ DOCKERMERGED.containers['so-elastalert'].ip }}
+        - ipv4_address: {{ DOCKER.containers['so-elastalert'].ip }}
    - detach: True
    - binds:
      - /opt/so/rules/elastalert:/opt/elastalert/rules/:ro
@@ -33,30 +33,24 @@ so-elastalert:
      - /opt/so/conf/elastalert/predefined/:/opt/elastalert/predefined/:ro
      - /opt/so/conf/elastalert/custom/:/opt/elastalert/custom/:ro
      - /opt/so/conf/elastalert/elastalert_config.yaml:/opt/elastalert/config.yaml:ro
-      {% if DOCKERMERGED.containers['so-elastalert'].custom_bind_mounts %}
-        {% for BIND in DOCKERMERGED.containers['so-elastalert'].custom_bind_mounts %}
+      {% if DOCKER.containers['so-elastalert'].custom_bind_mounts %}
+        {% for BIND in DOCKER.containers['so-elastalert'].custom_bind_mounts %}
      - {{ BIND }}
        {% endfor %}
      {% endif %}
    - extra_hosts:
      - {{ GLOBALS.manager }}:{{ GLOBALS.manager_ip }}
-      {% if DOCKERMERGED.containers['so-elastalert'].extra_hosts %}
-        {% for XTRAHOST in DOCKERMERGED.containers['so-elastalert'].extra_hosts %}
+      {% if DOCKER.containers['so-elastalert'].extra_hosts %}
+        {% for XTRAHOST in DOCKER.containers['so-elastalert'].extra_hosts %}
      - {{ XTRAHOST }}
        {% endfor %}
      {% endif %}
-      {% if DOCKERMERGED.containers['so-elastalert'].extra_env %}
+      {% if DOCKER.containers['so-elastalert'].extra_env %}
    - environment:
-        {% for XTRAENV in DOCKERMERGED.containers['so-elastalert'].extra_env %}
+        {% for XTRAENV in DOCKER.containers['so-elastalert'].extra_env %}
      - {{ XTRAENV }}
        {% endfor %}
      {% endif %}
-    {% if DOCKERMERGED.containers['so-elastalert'].ulimits %}
-    - ulimits:
-    {%   for ULIMIT in DOCKERMERGED.containers['so-elastalert'].ulimits %}
-      - {{ ULIMIT.name }}={{ ULIMIT.soft }}:{{ ULIMIT.hard }}
-    {%   endfor %}
-    {% endif %}
    - require:
      - cmd: wait_for_elasticsearch
      - file: elastarules
@@ -0,0 +1 @@
+THIS IS A PLACEHOLDER FILE
@@ -1,48 +1,47 @@
 elastalert:
  enabled:
    description: Enables or disables the ElastAlert 2 process. This process is critical for ensuring alerts arrive in SOC, and for outbound notification delivery.
-    forcedType: bool
-    helpLink: elastalert
+    helpLink: elastalert.html
  alerter_parameters:
    title: Custom Configuration Parameters
    description: Optional configuration parameters made available as defaults for all rules and alerters. Use YAML format for these parameters, and reference the ElastAlert 2 documentation, located at https://elastalert2.readthedocs.io, for available configuration parameters. Requires a valid Security Onion license key.
    global: True
    multiline: True
    syntax: yaml
-    helpLink: elastalert
+    helpLink: elastalert.html
    forcedType: string
  jira_api_key:
    title: Jira API Key
    description: Optional configuration parameter for Jira API Key, used instead of the Jira username and password. Requires a valid Security Onion license key.
    global: True
    sensitive: True
-    helpLink: elastalert
+    helpLink: elastalert.html
    forcedType: string
  jira_pass:
    title: Jira Password
    description: Optional configuration parameter for Jira password. Requires a valid Security Onion license key.
    global: True
    sensitive: True
-    helpLink: elastalert
+    helpLink: elastalert.html
    forcedType: string
  jira_user:
    title: Jira Username
    description: Optional configuration parameter for Jira username. Requires a valid Security Onion license key.
    global: True
-    helpLink: elastalert
+    helpLink: elastalert.html
    forcedType: string
  smtp_pass:
    title: SMTP Password
    description: Optional configuration parameter for SMTP password, required for authenticating email servers. Requires a valid Security Onion license key.
    global: True
    sensitive: True
-    helpLink: elastalert
+    helpLink: elastalert.html
    forcedType: string
  smtp_user:
    title: SMTP Username
    description: Optional configuration parameter for SMTP username, required for authenticating email servers. Requires a valid Security Onion license key.
    global: True
-    helpLink: elastalert
+    helpLink: elastalert.html
    forcedType: string
  files:
    custom:
@@ -50,131 +49,91 @@ elastalert:
        description: Optional custom Certificate Authority for connecting to an AlertManager server. To utilize this custom file, the alertmanager_ca_certs key must be set to /opt/elastalert/custom/alertmanager_ca.crt in the Alerter Parameters setting. Requires a valid Security Onion license key.
        global: True
        file: True
-        helpLink: elastalert
+        helpLink: elastalert.html
      gelf_ca__crt:
        description: Optional custom Certificate Authority for connecting to a Graylog server. To utilize this custom file, the graylog_ca_certs key must be set to /opt/elastalert/custom/graylog_ca.crt in the Alerter Parameters setting. Requires a valid Security Onion license key.
        global: True
        file: True
-        helpLink: elastalert
+        helpLink: elastalert.html
      http_post_ca__crt:
        description: Optional custom Certificate Authority for connecting to a generic HTTP server, via the legacy HTTP POST alerter. To utilize this custom file, the http_post_ca_certs key must be set to /opt/elastalert/custom/http_post2_ca.crt in the Alerter Parameters setting. Requires a valid Security Onion license key.
        global: True
        file: True
-        helpLink: elastalert
+        helpLink: elastalert.html
      http_post2_ca__crt:
        description: Optional custom Certificate Authority for connecting to a generic HTTP server, via the newer HTTP POST 2 alerter. To utilize this custom file, the http_post2_ca_certs key must be set to /opt/elastalert/custom/http_post2_ca.crt in the Alerter Parameters setting. Requires a valid Security Onion license key.
        global: True
        file: True
-        helpLink: elastalert
+        helpLink: elastalert.html
      ms_teams_ca__crt:
        description: Optional custom Certificate Authority for connecting to Microsoft Teams server. To utilize this custom file, the ms_teams_ca_certs key must be set to /opt/elastalert/custom/ms_teams_ca.crt in the Alerter Parameters setting. Requires a valid Security Onion license key.
        global: True
        file: True
-        helpLink: elastalert
+        helpLink: elastalert.html
      pagerduty_ca__crt:
        description: Optional custom Certificate Authority for connecting to PagerDuty server. To utilize this custom file, the pagerduty_ca_certs key must be set to /opt/elastalert/custom/pagerduty_ca.crt in the Alerter Parameters setting. Requires a valid Security Onion license key.
        global: True
        file: True
-        helpLink: elastalert
+        helpLink: elastalert.html
      rocket_chat_ca__crt:
        description: Optional custom Certificate Authority for connecting to PagerDuty server. To utilize this custom file, the rocket_chart_ca_certs key must be set to /opt/elastalert/custom/rocket_chat_ca.crt in the Alerter Parameters setting. Requires a valid Security Onion license key.
        global: True
        file: True
-        helpLink: elastalert
+        helpLink: elastalert.html
      smtp__crt:
        description: Optional custom certificate for connecting to an SMTP server. To utilize this custom file, the smtp_cert_file key must be set to /opt/elastalert/custom/smtp.crt in the Alerter Parameters setting. Requires a valid Security Onion license key.
        global: True
        file: True
-        helpLink: elastalert
+        helpLink: elastalert.html
      smtp__key:
        description: Optional custom certificate key for connecting to an SMTP server. To utilize this custom file, the smtp_key_file key must be set to /opt/elastalert/custom/smtp.key in the Alerter Parameters setting. Requires a valid Security Onion license key.
        global: True
        file: True
-        helpLink: elastalert
+        helpLink: elastalert.html
      slack_ca__crt:
        description: Optional custom Certificate Authority for connecting to Slack. To utilize this custom file, the slack_ca_certs key must be set to /opt/elastalert/custom/slack_ca.crt in the Alerter Parameters setting. Requires a valid Security Onion license key.
        global: True
        file: True
-        helpLink: elastalert
+        helpLink: elastalert.html
  config:
-    scan_subdirectories:
-      description: Recursively scan subdirectories for rules.
-      forcedType: bool
-      advanced: True
-      global: True
-      helpLink: elastalert
    disable_rules_on_error:
      description: Disable rules on failure.
-      forcedType: bool
      global: True
-      helpLink: elastalert
+      helpLink: elastalert.html
    run_every:
      minutes:
        description: Amount of time in minutes between searches.
        global: True
-        helpLink: elastalert
+        helpLink: elastalert.html
    buffer_time:
      minutes:
        description: Amount of time in minutes to look through.
        global: True
-        helpLink: elastalert
+        helpLink: elastalert.html
    old_query_limit:
      minutes:
        description: Amount of time in minutes between queries to start at the most recently run query.
        global: True
-        helpLink: elastalert
+        helpLink: elastalert.html
    es_conn_timeout:
      description: Timeout in seconds for connecting to and reading from Elasticsearch.
      global: True
-      helpLink: elastalert
+      helpLink: elastalert.html
    max_query_size:
      description: The maximum number of documents that will be returned from Elasticsearch in a single query.
      global: True
-      helpLink: elastalert
-    use_ssl:
-      description: Use SSL to connect to Elasticsearch.
-      forcedType: bool
-      advanced: True
-      global: True
-      helpLink: elastalert
-    verify_certs:
-      description: Verify TLS certificates when connecting to Elasticsearch.
-      forcedType: bool
-      advanced: True
-      global: True
-      helpLink: elastalert
+      helpLink: elastalert.html
    alert_time_limit:
      days:
        description: The retry window for failed alerts.
        global: True
-        helpLink: elastalert
+        helpLink: elastalert.html
    index_settings:
      shards:
        description: The number of shards for elastalert indices.
        global: True
-        helpLink: elastalert
+        helpLink: elastalert.html
      replicas:
        description: The number of replicas for elastalert indices.
        global: True
-        helpLink: elastalert
-    logging:
-      incremental:
-        description: When incremental is false (the default), the logging configuration is applied in full, replacing any existing logging setup. When true, only the level attributes of existing loggers and handlers are updated, leaving the rest of the logging configuration unchanged.
-        forcedType: bool
-        advanced: True
-        global: True
-        helpLink: elastalert
-      disable_existing_loggers:
-        description: Disable existing loggers.
-        forcedType: bool
-        advanced: True
-        global: True
-        helpLink: elastalert
-      loggers:
-        '':
-          propagate:
-            description: Propagate log messages to parent loggers.
-            forcedType: bool
-            advanced: True
-            global: True
-            helpLink: elastalert
+        helpLink: elastalert.html
@@ -6,7 +6,7 @@
 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
-{%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
+{%   from 'docker/docker.map.jinja' import DOCKER %}

 include:
  - elastic-fleet-package-registry.config
@@ -21,46 +21,30 @@ so-elastic-fleet-package-registry:
    - user: 948
    - networks:
      - sobridge:
-        - ipv4_address: {{ DOCKERMERGED.containers['so-elastic-fleet-package-registry'].ip }}
+        - ipv4_address: {{ DOCKER.containers['so-elastic-fleet-package-registry'].ip }}
    - extra_hosts:
        - {{ GLOBALS.hostname }}:{{ GLOBALS.node_ip }}
-        {% if DOCKERMERGED.containers['so-elastic-fleet-package-registry'].extra_hosts %}
-          {% for XTRAHOST in DOCKERMERGED.containers['so-elastic-fleet-package-registry'].extra_hosts %}
+        {% if DOCKER.containers['so-elastic-fleet-package-registry'].extra_hosts %}
+          {% for XTRAHOST in DOCKER.containers['so-elastic-fleet-package-registry'].extra_hosts %}
        - {{ XTRAHOST }}
          {% endfor %}
        {% endif %}
    - port_bindings:
-      {% for BINDING in DOCKERMERGED.containers['so-elastic-fleet-package-registry'].port_bindings %}
+      {% for BINDING in DOCKER.containers['so-elastic-fleet-package-registry'].port_bindings %}
      - {{ BINDING }}
      {% endfor %}
-    {% if DOCKERMERGED.containers['so-elastic-fleet-package-registry'].custom_bind_mounts %}
+    {% if DOCKER.containers['so-elastic-fleet-package-registry'].custom_bind_mounts %}
    - binds:
-        {% for BIND in DOCKERMERGED.containers['so-elastic-fleet-package-registry'].custom_bind_mounts %}
+        {% for BIND in DOCKER.containers['so-elastic-fleet-package-registry'].custom_bind_mounts %}
      - {{ BIND }}
        {% endfor %}
    {% endif %}
-    {% if DOCKERMERGED.containers['so-elastic-fleet-package-registry'].extra_env %}
+    {% if DOCKER.containers['so-elastic-fleet-package-registry'].extra_env %}
    - environment:
-        {% for XTRAENV in DOCKERMERGED.containers['so-elastic-fleet-package-registry'].extra_env %}
+        {% for XTRAENV in DOCKER.containers['so-elastic-fleet-package-registry'].extra_env %}
      - {{ XTRAENV }}
        {% endfor %}
      {% endif %}
-    {% if DOCKERMERGED.containers['so-elastic-fleet-package-registry'].ulimits %}
-    - ulimits:
-    {%   for ULIMIT in DOCKERMERGED.containers['so-elastic-fleet-package-registry'].ulimits %}
-      - {{ ULIMIT.name }}={{ ULIMIT.soft }}:{{ ULIMIT.hard }}
-    {%   endfor %}
-    {% endif %}
-
-wait_for_so-elastic-fleet-package-registry:
-  http.wait_for_successful_query:
-    - name: "http://localhost:8080/health"
-    - status: 200
-    - wait_for: 300
-    - request_interval: 15
-    - require:
-      - docker_container: so-elastic-fleet-package-registry
-
 delete_so-elastic-fleet-package-registry_so-status.disabled:
  file.uncomment:
    - name: /opt/so/conf/so-status/so-status.conf
@@ -1,5 +1,4 @@
 elastic_fleet_package_registry:
  enabled:
    description: Enables or disables the Fleet package registry process. This process must remain enabled to allow Elastic Agent packages to be updated.
-    forcedType: bool
    advanced: True
@@ -6,7 +6,7 @@
 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
-{%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
+{%   from 'docker/docker.map.jinja' import DOCKER %}

 include:
  - ca
@@ -22,17 +22,17 @@ so-elastic-agent:
    - user: 949
    - networks:
      - sobridge:
-        - ipv4_address: {{ DOCKERMERGED.containers['so-elastic-agent'].ip }}
+        - ipv4_address: {{ DOCKER.containers['so-elastic-agent'].ip }}
    - extra_hosts:
        - {{ GLOBALS.manager }}:{{ GLOBALS.manager_ip }}
        - {{ GLOBALS.hostname }}:{{ GLOBALS.node_ip }}
-        {% if DOCKERMERGED.containers['so-elastic-agent'].extra_hosts %}
-          {% for XTRAHOST in DOCKERMERGED.containers['so-elastic-agent'].extra_hosts %}
+        {% if DOCKER.containers['so-elastic-agent'].extra_hosts %}
+          {% for XTRAHOST in DOCKER.containers['so-elastic-agent'].extra_hosts %}
        - {{ XTRAHOST }}
          {% endfor %}
        {% endif %}
    - port_bindings:
-      {% for BINDING in DOCKERMERGED.containers['so-elastic-agent'].port_bindings %}
+      {% for BINDING in DOCKER.containers['so-elastic-agent'].port_bindings %}
      - {{ BINDING }}
      {% endfor %}
    - binds:
@@ -41,25 +41,19 @@ so-elastic-agent:
      - /etc/pki/tls/certs/intca.crt:/etc/pki/tls/certs/intca.crt:ro 
      - /nsm:/nsm:ro
      - /opt/so/log:/opt/so/log:ro
-     {% if DOCKERMERGED.containers['so-elastic-agent'].custom_bind_mounts %}
-        {% for BIND in DOCKERMERGED.containers['so-elastic-agent'].custom_bind_mounts %}
+     {% if DOCKER.containers['so-elastic-agent'].custom_bind_mounts %}
+        {% for BIND in DOCKER.containers['so-elastic-agent'].custom_bind_mounts %}
      - {{ BIND }}
        {% endfor %}
      {% endif %}
    - environment:
      - FLEET_CA=/etc/pki/tls/certs/intca.crt
      - LOGS_PATH=logs
-      {% if DOCKERMERGED.containers['so-elastic-agent'].extra_env %}
-        {% for XTRAENV in DOCKERMERGED.containers['so-elastic-agent'].extra_env %}
+      {% if DOCKER.containers['so-elastic-agent'].extra_env %}
+        {% for XTRAENV in DOCKER.containers['so-elastic-agent'].extra_env %}
      - {{ XTRAENV }}
        {% endfor %}
      {% endif %}
-    {% if DOCKERMERGED.containers['so-elastic-agent'].ulimits %}
-    - ulimits:
-    {%   for ULIMIT in DOCKERMERGED.containers['so-elastic-agent'].ulimits %}
-      - {{ ULIMIT.name }}={{ ULIMIT.soft }}:{{ ULIMIT.hard }}
-    {%   endfor %}
-    {% endif %}
    - require:
      - file: create-elastic-agent-config
      - file: trusttheca
@@ -1,5 +1,4 @@
 elasticagent:
  enabled:
    description: Enables or disables the Elastic Agent process. This process must remain enabled to allow collection of node events.
-    forcedType: bool
    advanced: True
@@ -1,123 +0,0 @@
-{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-  or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use
-  this file except in compliance with the Elastic License 2.0. #}
-
-
-{% import_json '/opt/so/state/esfleet_content_package_components.json' as ADDON_CONTENT_PACKAGE_COMPONENTS %}
-{% import_json '/opt/so/state/esfleet_component_templates.json' as INSTALLED_COMPONENT_TEMPLATES %}
-{% import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %}
-
-{% set CORE_ESFLEET_PACKAGES = ELASTICFLEETDEFAULTS.get('elasticfleet', {}).get('packages', {}) %}
-{% set ADDON_CONTENT_INTEGRATION_DEFAULTS = {} %}
-{% set DEBUG_STUFF = {} %}
-
-{% for pkg in ADDON_CONTENT_PACKAGE_COMPONENTS %}
-{%   if pkg.name in CORE_ESFLEET_PACKAGES %}
-{#     skip core content packages #}
-{%   elif pkg.name not in CORE_ESFLEET_PACKAGES %}
-{#     generate defaults for each content package #}
-{%     if pkg.dataStreams is defined and pkg.dataStreams is not none and pkg.dataStreams | length > 0%}
-{%       for pattern in pkg.dataStreams %}
-{#         in ES 9.3.2 'input' type integrations no longer create default component templates and instead they wait for user input during 'integration' setup (fleet ui config)
-             title: generic is an artifact of that and is not in use #}
-{%         if pattern.title == "generic" %}
-{%           continue %}
-{%         endif %}
-{%         if "metrics-" in pattern.name %}
-{%           set integration_type = "metrics-" %}
-{%         elif "logs-" in pattern.name %}
-{%           set integration_type = "logs-" %}
-{%         else %}
-{%           set integration_type = "" %}
-{%         endif %}
-{#         on content integrations the component name is user defined at the time it is added to an agent policy #}
-{%         set component_name = pattern.title %}
-{%         set index_pattern = pattern.name %}
-{#           component_name_x maintains the functionality of merging local pillar changes with generated 'defaults' via SOC UI #}
-{%           set component_name_x = component_name.replace(".","_x_") %}
-{#           pillar overrides/merge expects the key names to follow the naming in elasticsearch/defaults.yaml eg. so-logs-1password_x_item_usages . The _x_ is replaced later on in elasticsearch/template.map.jinja #}
-{%           set integration_key = "so-" ~ integration_type ~ pkg.name + '_x_' ~ component_name_x %}
-{#           Default integration settings #}
-{%           set integration_defaults = {
-               "index_sorting": false,
-               "index_template": {
-                 "composed_of": [integration_type ~ component_name ~ "@package", integration_type ~ component_name ~ "@custom", "so-fleet_integrations.ip_mappings-1", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"],
-                 "data_stream": {
-                   "allow_custom_routing": false,
-                   "hidden": false
-                 },
-                 "ignore_missing_component_templates": [integration_type ~ component_name ~ "@custom"],
-                 "index_patterns": [index_pattern],
-                 "priority": 501,
-                 "template": {
-                   "settings": {
-                     "index": {
-                       "lifecycle": {"name": "so-" ~ integration_type ~ component_name ~ "-logs"},
-                         "number_of_replicas": 0
-                     }
-                   }
-                 }
-               },
-               "policy": {
-                 "phases": {
-                   "cold": {
-                     "actions": {
-                        "allocate":{
-                          "number_of_replicas": ""
-                        },
-                       "set_priority": {"priority": 0}
-                     },
-                     "min_age": "60d"
-                   },
-                   "delete": {
-                     "actions": {
-                       "delete": {}
-                     },
-                     "min_age": "365d"
-                   },
-                   "hot": {
-                     "actions": {
-                       "rollover": {
-                         "max_age": "30d",
-                         "max_primary_shard_size": "50gb"
-                       },
-                       "forcemerge":{
-                         "max_num_segments": ""
-                       },
-                       "shrink":{
-                         "max_primary_shard_size": "",
-                         "method": "COUNT",
-                         "number_of_shards": ""
-                       },
-                       "set_priority": {"priority": 100}
-                      },
-                      "min_age": "0ms"
-                   },
-                   "warm": {
-                     "actions": {
-                        "allocate": {
-                          "number_of_replicas": ""
-                        },
-                        "forcemerge": {
-                          "max_num_segments": ""
-                        },
-                        "shrink":{
-                          "max_primary_shard_size": "",
-                          "method": "COUNT",
-                          "number_of_shards": ""
-                        },
-                       "set_priority": {"priority": 50}
-                     },
-                     "min_age": "30d"
-                   }
-                 }
-               }
-             } %}
-
-
-{%         do ADDON_CONTENT_INTEGRATION_DEFAULTS.update({integration_key: integration_defaults}) %}
-{%       endfor %}
-{%     else %}
-{%     endif %}
-{%   endif %}
-{% endfor %}
@@ -1,6 +1,5 @@
 elasticfleet:
  enabled: False
-  patch_version: 9.3.3+build202604082258 # Elastic Agent specific patch release.
  enable_manager_output: True
  config:
    server:
@@ -6,7 +6,7 @@
 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
-{%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
+{%   from 'docker/docker.map.jinja' import DOCKER %}
 {%   from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %}

 {#   This value is generated during node install and stored in minion pillar #}
@@ -17,17 +17,65 @@ include:
  - logstash.ssl
  - elasticfleet.config
  - elasticfleet.sostatus
-{%- if GLOBALS.role != "so-fleet" %}
-  - elasticfleet.manager
-{%- endif %}

-{% if GLOBALS.role != "so-fleet" %}
+{% if grains.role not in ['so-fleet'] %}
 # Wait for Elasticsearch to be ready - no reason to try running Elastic Fleet server if ES is not ready
 wait_for_elasticsearch_elasticfleet:
  cmd.run:
    - name: so-elasticsearch-wait
+{% endif %}
+
+# If enabled, automatically update Fleet Logstash Outputs
+{% if ELASTICFLEETMERGED.config.server.enable_auto_configuration and grains.role not in ['so-import', 'so-eval', 'so-fleet'] %}
+so-elastic-fleet-auto-configure-logstash-outputs:
+  cmd.run:
+    - name: /usr/sbin/so-elastic-fleet-outputs-update
+    - retry:
+        attempts: 4
+        interval: 30
+
+{# Separate from above in order to catch elasticfleet-logstash.crt changes and force update to fleet output policy #}
+so-elastic-fleet-auto-configure-logstash-outputs-force:
+  cmd.run:
+    - name: /usr/sbin/so-elastic-fleet-outputs-update --certs
+    - retry:
+        attempts: 4
+        interval: 30
+    - onchanges:
+        - x509: etc_elasticfleet_logstash_crt
+        - x509: elasticfleet_kafka_crt
+{% endif %}
+
+# If enabled, automatically update Fleet Server URLs & ES Connection
+{% if ELASTICFLEETMERGED.config.server.enable_auto_configuration and grains.role not in ['so-fleet'] %}
+so-elastic-fleet-auto-configure-server-urls:
+  cmd.run:
+    - name: /usr/sbin/so-elastic-fleet-urls-update
+    - retry:
+        attempts: 4
+        interval: 30
+{% endif %}
+
+# Automatically update Fleet Server Elasticsearch URLs & Agent Artifact URLs
+{% if grains.role not in ['so-fleet'] %}
+so-elastic-fleet-auto-configure-elasticsearch-urls:
+  cmd.run:
+    - name: /usr/sbin/so-elastic-fleet-es-url-update
+    - retry:
+        attempts: 4
+        interval: 30
+
+so-elastic-fleet-auto-configure-artifact-urls:
+  cmd.run:
+    - name: /usr/sbin/so-elastic-fleet-artifacts-url-update
+    - retry:
+        attempts: 4
+        interval: 30
+
+{% endif %}

 # Sync Elastic Agent artifacts to Fleet Node
+{% if grains.role in ['so-fleet'] %}
 elasticagent_syncartifacts:
  file.recurse:
    - name: /nsm/elastic-fleet/artifacts/beats
@@ -46,17 +94,17 @@ so-elastic-fleet:
    - user: 947
    - networks:
      - sobridge:
-        - ipv4_address: {{ DOCKERMERGED.containers['so-elastic-fleet'].ip }}
+        - ipv4_address: {{ DOCKER.containers['so-elastic-fleet'].ip }}
    - extra_hosts:
        - {{ GLOBALS.manager }}:{{ GLOBALS.manager_ip }}
        - {{ GLOBALS.hostname }}:{{ GLOBALS.node_ip }}
-        {% if DOCKERMERGED.containers['so-elastic-fleet'].extra_hosts %}
-          {% for XTRAHOST in DOCKERMERGED.containers['so-elastic-fleet'].extra_hosts %}
+        {% if DOCKER.containers['so-elastic-fleet'].extra_hosts %}
+          {% for XTRAHOST in DOCKER.containers['so-elastic-fleet'].extra_hosts %}
        - {{ XTRAHOST }}
          {% endfor %}
        {% endif %}
    - port_bindings:
-      {% for BINDING in DOCKERMERGED.containers['so-elastic-fleet'].port_bindings %}
+      {% for BINDING in DOCKER.containers['so-elastic-fleet'].port_bindings %}
      - {{ BINDING }}
      {% endfor %}
    - binds:
@@ -64,8 +112,8 @@ so-elastic-fleet:
      - /etc/pki/elasticfleet-server.key:/etc/pki/elasticfleet-server.key:ro
      - /etc/pki/tls/certs/intca.crt:/etc/pki/tls/certs/intca.crt:ro
      - /opt/so/log/elasticfleet:/usr/share/elastic-agent/logs 
-     {% if DOCKERMERGED.containers['so-elastic-fleet'].custom_bind_mounts %}
-        {% for BIND in DOCKERMERGED.containers['so-elastic-fleet'].custom_bind_mounts %}
+     {% if DOCKER.containers['so-elastic-fleet'].custom_bind_mounts %}
+        {% for BIND in DOCKER.containers['so-elastic-fleet'].custom_bind_mounts %}
      - {{ BIND }}
        {% endfor %}
      {% endif %}      
@@ -80,17 +128,11 @@ so-elastic-fleet:
      - FLEET_CA=/etc/pki/tls/certs/intca.crt     
      - FLEET_SERVER_ELASTICSEARCH_CA=/etc/pki/tls/certs/intca.crt
      - LOGS_PATH=logs
-      {% if DOCKERMERGED.containers['so-elastic-fleet'].extra_env %}
-        {% for XTRAENV in DOCKERMERGED.containers['so-elastic-fleet'].extra_env %}
+      {% if DOCKER.containers['so-elastic-fleet'].extra_env %}
+        {% for XTRAENV in DOCKER.containers['so-elastic-fleet'].extra_env %}
      - {{ XTRAENV }}
        {% endfor %}
      {% endif %}
-    {% if DOCKERMERGED.containers['so-elastic-fleet'].ulimits %}
-    - ulimits:
-    {%   for ULIMIT in DOCKERMERGED.containers['so-elastic-fleet'].ulimits %}
-      - {{ ULIMIT.name }}={{ ULIMIT.soft }}:{{ ULIMIT.hard }}
-    {%   endfor %}
-    {% endif %}
    - watch:
      - file: trusttheca
      - x509: etc_elasticfleet_key
@@ -101,6 +143,57 @@ so-elastic-fleet:
      - x509: etc_elasticfleet_crt
 {%   endif %}

+{%  if GLOBALS.role != "so-fleet" %}
+so-elastic-fleet-package-statefile:
+  file.managed:
+    - name: /opt/so/state/elastic_fleet_packages.txt
+    - contents: {{ELASTICFLEETMERGED.packages}}
+
+so-elastic-fleet-package-upgrade:
+  cmd.run:
+    - name: /usr/sbin/so-elastic-fleet-package-upgrade
+    - retry:
+        attempts: 3
+        interval: 10
+    - onchanges:
+      - file: /opt/so/state/elastic_fleet_packages.txt
+
+so-elastic-fleet-integrations:
+  cmd.run:
+    - name: /usr/sbin/so-elastic-fleet-integration-policy-load
+    - retry:
+        attempts: 3
+        interval: 10
+
+so-elastic-agent-grid-upgrade:
+  cmd.run:
+    - name: /usr/sbin/so-elastic-agent-grid-upgrade
+    - retry:
+        attempts: 12
+        interval: 5
+
+so-elastic-fleet-integration-upgrade:
+  cmd.run:
+    - name: /usr/sbin/so-elastic-fleet-integration-upgrade
+    - retry:
+        attempts: 3
+        interval: 10
+
+{# Optional integrations script doesn't need the retries like so-elastic-fleet-integration-upgrade which loads the default integrations #}
+so-elastic-fleet-addon-integrations:
+  cmd.run:
+    - name: /usr/sbin/so-elastic-fleet-optional-integrations-load
+
+{%   if ELASTICFLEETMERGED.config.defend_filters.enable_auto_configuration %}
+so-elastic-defend-manage-filters-file-watch:
+  cmd.run:
+    - name: python3 /sbin/so-elastic-defend-manage-filters.py -c /opt/so/conf/elasticsearch/curl.config -d /opt/so/conf/elastic-fleet/defend-exclusions/disabled-filters.yaml -i /nsm/securityonion-resources/event_filters/ -i /opt/so/conf/elastic-fleet/defend-exclusions/rulesets/custom-filters/ &>> /opt/so/log/elasticfleet/elastic-defend-manage-filters.log
+    - onchanges:
+      - file: elasticdefendcustom
+      - file: elasticdefenddisabled
+{%    endif %}
+{%  endif %}
+
 delete_so-elastic-fleet_so-status.disabled:
  file.uncomment:
    - name: /opt/so/conf/so-status/so-status.conf
@@ -9,22 +9,16 @@
  "namespace": "so",
  "description": "Zeek Import logs",
  "policy_id": "so-grid-nodes_general",
-  "policy_ids": [
-    "so-grid-nodes_general"
-  ],
-  "vars": {},
  "inputs": {
    "filestream-filestream": {
      "enabled": true,
      "streams": {
-        "filestream.filestream": {
+        "filestream.generic": {
          "enabled": true,
          "vars": {
            "paths": [
              "/nsm/import/*/zeek/logs/*.log"
            ],
-            "compression_gzip": false,
-            "use_logs_stream": false,
            "data_stream.dataset": "import",
            "pipeline": "",
            "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -40,8 +34,7 @@
            "fingerprint_length": "64",
            "file_identity_native": true,
            "exclude_lines": [],
-            "include_lines": [],
-            "delete_enabled": false
+            "include_lines": []
          }
        }
      }
@@ -15,25 +15,19 @@
    "version": ""
  },
  "name": "kratos-logs",
-  "namespace": "so",
  "description": "Kratos logs",
  "policy_id": "so-grid-nodes_general",
-  "policy_ids": [
-    "so-grid-nodes_general"
-  ],
-  "vars": {},
+  "namespace": "so",
  "inputs": {
    "filestream-filestream": {
      "enabled": true,
      "streams": {
-        "filestream.filestream": {
+        "filestream.generic": {
          "enabled": true,
          "vars": {
            "paths": [
              "/opt/so/log/kratos/kratos.log"
            ],
-            "compression_gzip": false,
-            "use_logs_stream": false,
            "data_stream.dataset": "kratos",
            "pipeline": "kratos",
            "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -54,10 +48,10 @@
            "harvester_limit": 0,
            "fingerprint": false,
            "fingerprint_offset": 0,
+            "fingerprint_length": "64",
            "file_identity_native": true,
            "exclude_lines": [],
-            "include_lines": [],
-            "delete_enabled": false
+            "include_lines": []
          }
        }
      }
@@ -9,22 +9,16 @@
  "namespace": "so",
  "description": "Zeek logs",
  "policy_id": "so-grid-nodes_general",
-  "policy_ids": [
-    "so-grid-nodes_general"
-  ],
-  "vars": {},
  "inputs": {
    "filestream-filestream": {
      "enabled": true,
      "streams": {
-        "filestream.filestream": {
+        "filestream.generic": {
          "enabled": true,
          "vars": {
            "paths": [
              "/nsm/zeek/logs/current/*.log"
            ],
-            "compression_gzip": false,
-            "use_logs_stream": false,
            "data_stream.dataset": "zeek",
            "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
            "exclude_files": ["({%- endraw -%}{{ ELASTICFLEETMERGED.logging.zeek.excluded | join('|') }}{%- raw -%})(\\..+)?\\.log$"],
@@ -36,10 +30,10 @@
            "harvester_limit": 0,
            "fingerprint": false,
            "fingerprint_offset": 0,
+            "fingerprint_length": "64",
            "file_identity_native": true,
            "exclude_lines": [],
-            "include_lines": [],
-            "delete_enabled": false
+            "include_lines": []
          }
        }
      }
@@ -5,7 +5,7 @@
  "package": {
    "name": "endpoint",
    "title": "Elastic Defend",
-    "version": "9.3.0",
+    "version": "9.0.2",
    "requires_root": true
  },
  "enabled": true,
@@ -6,23 +6,21 @@
  "name": "agent-monitor",
  "namespace": "",
  "description": "",
-  "policy_id": "so-grid-nodes_general",
  "policy_ids": [
    "so-grid-nodes_general"
  ],
+  "output_id": null,
  "vars": {},
  "inputs": {
    "filestream-filestream": {
      "enabled": true,
      "streams": {
-        "filestream.filestream": {
+        "filestream.generic": {
          "enabled": true,
          "vars": {
            "paths": [
              "/opt/so/log/agents/agent-monitor.log"
            ],
-            "compression_gzip": false,
-            "use_logs_stream": false,
            "data_stream.dataset": "agentmonitor",
            "pipeline": "elasticagent.monitor",
            "parsers": "",
@@ -36,16 +34,15 @@
            "ignore_older": "72h",
            "clean_inactive": -1,
            "harvester_limit": 0,
-            "fingerprint": false,
+            "fingerprint": true,
            "fingerprint_offset": 0,
-            "file_identity_native": true,
+            "fingerprint_length": 64,
+            "file_identity_native": false,
            "exclude_lines": [],
-            "include_lines": [],
-            "delete_enabled": false
+            "include_lines": []
          }
        }
      }
    }
-  },
-  "force": true
+  }
 }
@@ -4,25 +4,19 @@
    "version": ""
  },
  "name": "hydra-logs",
-  "namespace": "so",
  "description": "Hydra logs",
  "policy_id": "so-grid-nodes_general",
-  "policy_ids": [
-    "so-grid-nodes_general"
-  ],
-  "vars": {},
+  "namespace": "so",
  "inputs": {
    "filestream-filestream": {
      "enabled": true,
      "streams": {
-        "filestream.filestream": {
+        "filestream.generic": {
          "enabled": true,
          "vars": {
            "paths": [
              "/opt/so/log/hydra/hydra.log"
            ],
-            "compression_gzip": false,
-            "use_logs_stream": false,
            "data_stream.dataset": "hydra",
            "pipeline": "hydra",
            "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -40,10 +34,10 @@
            "harvester_limit": 0,
            "fingerprint": false,
            "fingerprint_offset": 0,
+            "fingerprint_length": "64",
            "file_identity_native": true,
            "exclude_lines": [],
-            "include_lines": [],
-            "delete_enabled": false
+            "include_lines": []
          }
        }
      }
@@ -4,25 +4,19 @@
    "version": ""
  },
  "name": "idh-logs",
-  "namespace": "so",
  "description": "IDH integration",
  "policy_id": "so-grid-nodes_general",
-  "policy_ids": [
-    "so-grid-nodes_general"
-  ],
-  "vars": {},
+  "namespace": "so",
  "inputs": {
    "filestream-filestream": {
      "enabled": true,
      "streams": {
-        "filestream.filestream": {
+        "filestream.generic": {
          "enabled": true,
          "vars": {
            "paths": [
              "/nsm/idh/opencanary.log"
            ],
-            "compression_gzip": false,
-            "use_logs_stream": false,
            "data_stream.dataset": "idh",
            "pipeline": "common",
            "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -37,10 +31,10 @@
            "harvester_limit": 0,
            "fingerprint": false,
            "fingerprint_offset": 0,
+            "fingerprint_length": "64",
            "file_identity_native": true,
            "exclude_lines": [],
-            "include_lines": [],
-            "delete_enabled": false
+            "include_lines": []
          }
        }
      }
@@ -4,32 +4,26 @@
    "version": ""
  },
  "name": "import-evtx-logs",
-  "namespace": "so",
  "description": "Import Windows EVTX logs",
  "policy_id": "so-grid-nodes_general",
-  "policy_ids": [
-    "so-grid-nodes_general"
-  ],
-  "vars": {},
+  "namespace": "so",
  "inputs": {
    "filestream-filestream": {
      "enabled": true,
      "streams": {
-        "filestream.filestream": {
+        "filestream.generic": {
          "enabled": true,
          "vars": {
            "paths": [
              "/nsm/import/*/evtx/*.json"
            ],
-            "compression_gzip": false,
-            "use_logs_stream": false,
            "data_stream.dataset": "import",
            "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
            "exclude_files": [
              "\\.gz$"
            ],
            "include_files": [],
-            "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- drop_fields:\n    fields: [\"host\"]\n    ignore_missing: true\n- add_fields:\n    target: data_stream\n    fields:\n      type: logs\n      dataset: system.security\n- add_fields:\n    target: event\n    fields:\n      dataset: system.security\n      module: system\n      imported: true\n- add_fields:\n    target: \"@metadata\"\n    fields:\n        pipeline: logs-system.security-2.15.0\n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.sysmon_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.sysmon_operational\n          module: windows\n          imported: true\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.sysmon_operational-3.8.0\n- if:\n    equals:\n      winlog.channel: 'Application'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.application-2.15.0\n- if:\n    equals:\n      winlog.channel: 'System'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.system-2.15.0\n    \n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.powershell_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell_operational\n          module: windows\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.powershell_operational-3.8.0\n- add_fields:\n    target: data_stream\n    fields:\n        dataset: import",
+            "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- drop_fields:\n    fields: [\"host\"]\n    ignore_missing: true\n- add_fields:\n    target: data_stream\n    fields:\n      type: logs\n      dataset: system.security\n- add_fields:\n    target: event\n    fields:\n      dataset: system.security\n      module: system\n      imported: true\n- add_fields:\n    target: \"@metadata\"\n    fields:\n        pipeline: logs-system.security-2.6.1\n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.sysmon_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.sysmon_operational\n          module: windows\n          imported: true\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.sysmon_operational-3.1.2\n- if:\n    equals:\n      winlog.channel: 'Application'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.application-2.6.1\n- if:\n    equals:\n      winlog.channel: 'System'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.system-2.6.1\n    \n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.powershell_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell_operational\n          module: windows\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.powershell_operational-3.1.2\n- add_fields:\n    target: data_stream\n    fields:\n        dataset: import",
            "tags": [
              "import"
            ],
@@ -39,10 +33,10 @@
            "harvester_limit": 0,
            "fingerprint": false,
            "fingerprint_offset": 0,
+            "fingerprint_length": "64",
            "file_identity_native": true,
            "exclude_lines": [],
-            "include_lines": [],
-            "delete_enabled": false
+            "include_lines": []
          }
        }
      }
@@ -4,25 +4,19 @@
    "version": ""
  },
  "name": "import-suricata-logs",
-  "namespace": "so",
  "description": "Import Suricata logs",
  "policy_id": "so-grid-nodes_general",
-  "policy_ids": [
-    "so-grid-nodes_general"
-  ],
-  "vars": {},
+  "namespace": "so",
  "inputs": {
    "filestream-filestream": {
      "enabled": true,
      "streams": {
-        "filestream.filestream": {
+        "filestream.generic": {
          "enabled": true,
          "vars": {
            "paths": [
              "/nsm/import/*/suricata/eve*.json"
            ],
-            "compression_gzip": false,
-            "use_logs_stream": false,
            "data_stream.dataset": "import",
            "pipeline": "suricata.common",
            "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -38,10 +32,10 @@
            "harvester_limit": 0,
            "fingerprint": false,
            "fingerprint_offset": 0,
+            "fingerprint_length": "64",
            "file_identity_native": true,
            "exclude_lines": [],
-            "include_lines": [],
-            "delete_enabled": false
+            "include_lines": []
          }
        }
      }
@@ -4,18 +4,14 @@
    "version": ""
  },
  "name": "rita-logs",
-  "namespace": "so",
  "description": "RITA Logs",
  "policy_id": "so-grid-nodes_general",
-  "policy_ids": [
-    "so-grid-nodes_general"
-  ],
-  "vars": {},
+  "namespace": "so",
  "inputs": {
    "filestream-filestream": {
      "enabled": true,
      "streams": {
-        "filestream.filestream": {
+        "filestream.generic": {
          "enabled": true,
          "vars": {
            "paths": [
@@ -23,8 +19,6 @@
              "/nsm/rita/exploded-dns.csv",
              "/nsm/rita/long-connections.csv"
            ],
-            "compression_gzip": false,
-            "use_logs_stream": false,
            "data_stream.dataset": "rita",
            "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
            "exclude_files": [
@@ -39,10 +33,10 @@
            "harvester_limit": 0,
            "fingerprint": false,
            "fingerprint_offset": 0,
+            "fingerprint_length": "64",
            "file_identity_native": true,
            "exclude_lines": [],
-            "include_lines": [],
-            "delete_enabled": false
+            "include_lines": []
          }
        }
      }
@@ -4,25 +4,19 @@
    "version": ""
  },
  "name": "so-ip-mappings",
-  "namespace": "so",
  "description": "IP Description mappings",
  "policy_id": "so-grid-nodes_general",
-  "policy_ids": [
-    "so-grid-nodes_general"
-  ],
-  "vars": {},
+  "namespace": "so",
  "inputs": {
    "filestream-filestream": {
      "enabled": true,
      "streams": {
-        "filestream.filestream": {
+        "filestream.generic": {
          "enabled": true,
          "vars": {
            "paths": [
              "/nsm/custom-mappings/ip-descriptions.csv"
            ],
-            "compression_gzip": false,
-            "use_logs_stream": false,
            "data_stream.dataset": "hostnamemappings",
            "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
            "exclude_files": [
@@ -38,10 +32,10 @@
            "harvester_limit": 0,
            "fingerprint": false,
            "fingerprint_offset": 0,
+            "fingerprint_length": "64",
            "file_identity_native": true,
            "exclude_lines": [],
-            "include_lines": [],
-            "delete_enabled": false
+            "include_lines": []
          }
        }
      }
@@ -4,25 +4,19 @@
    "version": ""
  },
  "name": "soc-auth-sync-logs",
-  "namespace": "so",
  "description": "Security Onion - Elastic Auth Sync - Logs",
  "policy_id": "so-grid-nodes_general",
-  "policy_ids": [
-    "so-grid-nodes_general"
-  ],
-  "vars": {},
+  "namespace": "so",
  "inputs": {
    "filestream-filestream": {
      "enabled": true,
      "streams": {
-        "filestream.filestream": {
+        "filestream.generic": {
          "enabled": true,
          "vars": {
            "paths": [
              "/opt/so/log/soc/sync.log"
            ],
-            "compression_gzip": false,
-            "use_logs_stream": false,
            "data_stream.dataset": "soc",
            "pipeline": "common",
            "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -37,10 +31,10 @@
            "harvester_limit": 0,
            "fingerprint": false,
            "fingerprint_offset": 0,
+            "fingerprint_length": "64",
            "file_identity_native": true,
            "exclude_lines": [],
-            "include_lines": [],
-            "delete_enabled": false
+            "include_lines": []
          }
        }
      }
@@ -4,26 +4,20 @@
    "version": ""
  },
  "name": "soc-detections-logs",
-  "namespace": "so",
  "description": "Security Onion Console - Detections Logs",
  "policy_id": "so-grid-nodes_general",
-  "policy_ids": [
-    "so-grid-nodes_general"
-  ],
-  "vars": {},
+  "namespace": "so",
  "inputs": {
    "filestream-filestream": {
      "enabled": true,
      "streams": {
-        "filestream.filestream": {
+        "filestream.generic": {
          "enabled": true,
          "vars": {
            "paths": [
              "/opt/so/log/soc/detections_runtime-status_sigma.log",
              "/opt/so/log/soc/detections_runtime-status_yara.log"
            ],
-            "compression_gzip": false,
-            "use_logs_stream": false,
            "data_stream.dataset": "soc",
            "pipeline": "common",
            "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -41,10 +35,10 @@
            "harvester_limit": 0,
            "fingerprint": false,
            "fingerprint_offset": 0,
+            "fingerprint_length": "64",
            "file_identity_native": true,
            "exclude_lines": [],
-            "include_lines": [],
-            "delete_enabled": false
+            "include_lines": []
          }
        }
      }
@@ -4,25 +4,19 @@
    "version": ""
  },
  "name": "soc-salt-relay-logs",
-  "namespace": "so",
  "description": "Security Onion - Salt Relay - Logs",
  "policy_id": "so-grid-nodes_general",
-  "policy_ids": [
-    "so-grid-nodes_general"
-  ],
-  "vars": {},
+  "namespace": "so",
  "inputs": {
    "filestream-filestream": {
      "enabled": true,
      "streams": {
-        "filestream.filestream": {
+        "filestream.generic": {
          "enabled": true,
          "vars": {
            "paths": [
              "/opt/so/log/soc/salt-relay.log"
            ],
-            "compression_gzip": false,
-            "use_logs_stream": false,
            "data_stream.dataset": "soc",
            "pipeline": "common",
            "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -39,10 +33,10 @@
            "harvester_limit": 0,
            "fingerprint": false,
            "fingerprint_offset": 0,
+            "fingerprint_length": "64",
            "file_identity_native": true,
            "exclude_lines": [],
-            "include_lines": [],
-            "delete_enabled": false
+            "include_lines": []
          }
        }
      }
@@ -4,25 +4,19 @@
    "version": ""
  },
  "name": "soc-sensoroni-logs",
-  "namespace": "so",
  "description": "Security Onion - Sensoroni - Logs",
  "policy_id": "so-grid-nodes_general",
-  "policy_ids": [
-    "so-grid-nodes_general"
-  ],
-  "vars": {},
+  "namespace": "so",
  "inputs": {
    "filestream-filestream": {
      "enabled": true,
      "streams": {
-        "filestream.filestream": {
+        "filestream.generic": {
          "enabled": true,
          "vars": {
            "paths": [
              "/opt/so/log/sensoroni/sensoroni.log"
            ],
-            "compression_gzip": false,
-            "use_logs_stream": false,
            "data_stream.dataset": "soc",
            "pipeline": "common",
            "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -37,10 +31,10 @@
            "harvester_limit": 0,
            "fingerprint": false,
            "fingerprint_offset": 0,
+            "fingerprint_length": "64",
            "file_identity_native": true,
            "exclude_lines": [],
-            "include_lines": [],
-            "delete_enabled": false
+            "include_lines": []
          }
        }
      }
@@ -4,25 +4,19 @@
    "version": ""
  },
  "name": "soc-server-logs",
-  "namespace": "so",
  "description": "Security Onion Console Logs",
  "policy_id": "so-grid-nodes_general",
-  "policy_ids": [
-    "so-grid-nodes_general"
-  ],
-  "vars": {},
+  "namespace": "so",
  "inputs": {
    "filestream-filestream": {
      "enabled": true,
      "streams": {
-        "filestream.filestream": {
+        "filestream.generic": {
          "enabled": true,
          "vars": {
            "paths": [
              "/opt/so/log/soc/sensoroni-server.log"
            ],
-            "compression_gzip": false,
-            "use_logs_stream": false,
            "data_stream.dataset": "soc",
            "pipeline": "common",
            "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -39,10 +33,10 @@
            "harvester_limit": 0,
            "fingerprint": false,
            "fingerprint_offset": 0,
+            "fingerprint_length": "64",
            "file_identity_native": true,
            "exclude_lines": [],
-            "include_lines": [],
-            "delete_enabled": false
+            "include_lines": []
          }
        }
      }
@@ -4,25 +4,19 @@
    "version": ""
  },
  "name": "strelka-logs",
-  "namespace": "so",
  "description": "Strelka Logs",
  "policy_id": "so-grid-nodes_general",
-  "policy_ids": [
-    "so-grid-nodes_general"
-  ],
-  "vars": {},
+  "namespace": "so",
  "inputs": {
    "filestream-filestream": {
      "enabled": true,
      "streams": {
-        "filestream.filestream": {
+        "filestream.generic": {
          "enabled": true,
          "vars": {
            "paths": [
              "/nsm/strelka/log/strelka.log"
            ],
-            "compression_gzip": false,
-            "use_logs_stream": false,
            "data_stream.dataset": "strelka",
            "pipeline": "strelka.file",
            "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -37,10 +31,10 @@
            "harvester_limit": 0,
            "fingerprint": false,
            "fingerprint_offset": 0,
+            "fingerprint_length": "64",
            "file_identity_native": true,
            "exclude_lines": [],
-            "include_lines": [],
-            "delete_enabled": false
+            "include_lines": []
          }
        }
      }
@@ -4,25 +4,19 @@
    "version": ""
  },
  "name": "suricata-logs",
-  "namespace": "so",
  "description": "Suricata integration",
  "policy_id": "so-grid-nodes_general",
-  "policy_ids": [
-    "so-grid-nodes_general"
-  ],
-  "vars": {},
+  "namespace": "so",
  "inputs": {
    "filestream-filestream": {
      "enabled": true,
      "streams": {
-        "filestream.filestream": {
+        "filestream.generic": {
          "enabled": true,
          "vars": {
            "paths": [
              "/nsm/suricata/eve*.json"
            ],
-            "compression_gzip": false,
-            "use_logs_stream": false,
            "data_stream.dataset": "suricata",
            "pipeline": "suricata.common",
            "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -37,10 +31,10 @@
            "harvester_limit": 0,
            "fingerprint": false,
            "fingerprint_offset": 0,
+            "fingerprint_length": "64",
            "file_identity_native": true,
            "exclude_lines": [],
-            "include_lines": [],
-            "delete_enabled": false
+            "include_lines": []
          }
        }
      }
@@ -1,123 +0,0 @@
-{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-  or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use
-  this file except in compliance with the Elastic License 2.0. #}
-
-
-{% import_json '/opt/so/state/esfleet_input_package_components.json' as ADDON_INPUT_PACKAGE_COMPONENTS %}
-{% import_json '/opt/so/state/esfleet_component_templates.json' as INSTALLED_COMPONENT_TEMPLATES %}
-{% import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %}
-
-{% set CORE_ESFLEET_PACKAGES = ELASTICFLEETDEFAULTS.get('elasticfleet', {}).get('packages', {}) %}
-{% set ADDON_INPUT_INTEGRATION_DEFAULTS = {} %}
-{% set DEBUG_STUFF = {} %}
-
-{% for pkg in ADDON_INPUT_PACKAGE_COMPONENTS %}
-{%   if pkg.name in CORE_ESFLEET_PACKAGES %}
-{#     skip core input packages #}
-{%   elif pkg.name not in CORE_ESFLEET_PACKAGES %}
-{#     generate defaults for each input package #}
-{%     if pkg.dataStreams is defined and pkg.dataStreams is not none and pkg.dataStreams | length > 0 %}
-{%       for pattern in pkg.dataStreams %}
-{#         in ES 9.3.2 'input' type integrations no longer create default component templates and instead they wait for user input during 'integration' setup (fleet ui config)
-             title: generic is an artifact of that and is not in use #}
-{%         if pattern.title == "generic" %}
-{%           continue %}
-{%         endif %}
-{%         if "metrics-" in pattern.name %}
-{%           set integration_type = "metrics-" %}
-{%         elif "logs-" in pattern.name %}
-{%           set integration_type = "logs-" %}
-{%         else %}
-{%           set integration_type = "" %}
-{%         endif %}
-{#         on input integrations the component name is user defined at the time it is added to an agent policy #}
-{%         set component_name = pattern.title %}
-{%         set index_pattern = pattern.name %}
-{#           component_name_x maintains the functionality of merging local pillar changes with generated 'defaults' via SOC UI #}
-{%           set component_name_x = component_name.replace(".","_x_") %}
-{#           pillar overrides/merge expects the key names to follow the naming in elasticsearch/defaults.yaml eg. so-logs-1password_x_item_usages . The _x_ is replaced later on in elasticsearch/template.map.jinja #}
-{%           set integration_key = "so-" ~ integration_type ~ pkg.name + '_x_' ~ component_name_x %}
-{#           Default integration settings #}
-{%           set integration_defaults = {
-               "index_sorting": false,
-               "index_template": {
-                 "composed_of": [integration_type ~ component_name ~ "@package", integration_type ~ component_name ~ "@custom", "so-fleet_integrations.ip_mappings-1", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"],
-                 "data_stream": {
-                   "allow_custom_routing": false,
-                   "hidden": false
-                 },
-                 "ignore_missing_component_templates": [integration_type ~ component_name ~ "@custom"],
-                 "index_patterns": [index_pattern],
-                 "priority": 501,
-                 "template": {
-                   "settings": {
-                     "index": {
-                       "lifecycle": {"name": "so-" ~ integration_type ~ component_name ~ "-logs"},
-                         "number_of_replicas": 0
-                     }
-                   }
-                 }
-               },
-               "policy": {
-                 "phases": {
-                   "cold": {
-                     "actions": {
-                        "allocate":{
-                          "number_of_replicas": ""
-                        },
-                       "set_priority": {"priority": 0}
-                     },
-                     "min_age": "60d"
-                   },
-                   "delete": {
-                     "actions": {
-                       "delete": {}
-                     },
-                     "min_age": "365d"
-                   },
-                   "hot": {
-                     "actions": {
-                       "rollover": {
-                         "max_age": "30d",
-                         "max_primary_shard_size": "50gb"
-                       },
-                       "forcemerge":{
-                         "max_num_segments": ""
-                       },
-                       "shrink":{
-                         "max_primary_shard_size": "",
-                         "method": "COUNT",
-                         "number_of_shards": ""
-                       },
-                       "set_priority": {"priority": 100}
-                      },
-                      "min_age": "0ms"
-                   },
-                   "warm": {
-                     "actions": {
-                        "allocate": {
-                          "number_of_replicas": ""
-                        },
-                        "forcemerge": {
-                          "max_num_segments": ""
-                        },
-                        "shrink":{
-                          "max_primary_shard_size": "",
-                          "method": "COUNT",
-                          "number_of_shards": ""
-                        },
-                       "set_priority": {"priority": 50}
-                     },
-                     "min_age": "30d"
-                   }
-                 }
-               }
-             } %}
-
-
-{%         do ADDON_INPUT_INTEGRATION_DEFAULTS.update({integration_key: integration_defaults}) %}
-{%         do DEBUG_STUFF.update({integration_key: "Generating defaults for "+ pkg.name })%}
-{%       endfor %}
-{%     endif %}
-{%   endif %}
-{% endfor %}
@@ -59,8 +59,8 @@
 {#     skip core integrations #}
 {%   elif pkg.name not in CORE_ESFLEET_PACKAGES %}
 {#     generate defaults for each integration #}
-{%     if pkg.dataStreams is defined and pkg.dataStreams is not none and pkg.dataStreams | length > 0 %}
-{%       for pattern in pkg.dataStreams %}
+{%     if pkg.es_index_patterns is defined and pkg.es_index_patterns is not none %}
+{%       for pattern in pkg.es_index_patterns %}
 {%         if "metrics-" in pattern.name %}
 {%           set integration_type = "metrics-" %}
 {%         elif "logs-" in pattern.name %}
@@ -75,27 +75,44 @@
 {%         if component_name in WEIRD_INTEGRATIONS %}
 {%           set component_name = WEIRD_INTEGRATIONS[component_name] %}
 {%         endif %}
+
+{#         create duplicate of component_name, so we can split generics from @custom component templates in the index template below and overwrite the default @package when needed
+           eg. having to replace unifiedlogs.generic@package with filestream.generic@package, but keep the ability to customize unifiedlogs.generic@custom and its ILM policy #}
+{%         set custom_component_name = component_name %}
+
+{#         duplicate integration_type to assist with sometimes needing to overwrite component templates with 'logs-filestream.generic@package' (there is no metrics-filestream.generic@package) #}
+{%         set generic_integration_type = integration_type %}
+
 {#           component_name_x maintains the functionality of merging local pillar changes with generated 'defaults' via SOC UI #}
 {%           set component_name_x = component_name.replace(".","_x_") %}
 {#           pillar overrides/merge expects the key names to follow the naming in elasticsearch/defaults.yaml eg. so-logs-1password_x_item_usages . The _x_ is replaced later on in elasticsearch/template.map.jinja #}
 {%           set integration_key = "so-" ~ integration_type ~ component_name_x %}

+{#         if its a .generic template make sure that a .generic@package for the integration exists. Else default to logs-filestream.generic@package #}
+{%         if ".generic" in component_name and integration_type ~ component_name ~ "@package" not in INSTALLED_COMPONENT_TEMPLATES %}
+{#           these generic templates by default are directed to index_pattern of 'logs-generic-*', overwrite that here to point to eg gcp_pubsub.generic-* #}
+{%           set index_pattern = integration_type ~ component_name ~ "-*" %}
+{#           includes use of .generic component template, but it doesn't exist in installed component templates. Redirect it to filestream.generic@package #}
+{%           set component_name = "filestream.generic" %}
+{%           set generic_integration_type = "logs-" %}
+{%         endif %}
+
 {#           Default integration settings #}
 {%           set integration_defaults = {
               "index_sorting": false,
               "index_template": {
-                 "composed_of": [integration_type ~ component_name ~ "@package", integration_type ~ component_name ~ "@custom", "so-fleet_integrations.ip_mappings-1", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"],
+                 "composed_of": [generic_integration_type ~ component_name ~ "@package", integration_type ~ custom_component_name ~ "@custom", "so-fleet_integrations.ip_mappings-1", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"],
                 "data_stream": {
                   "allow_custom_routing": false,
                   "hidden": false
                 },
-                 "ignore_missing_component_templates": [integration_type ~ component_name ~ "@custom"],
+                 "ignore_missing_component_templates": [integration_type ~ custom_component_name ~ "@custom"],
                 "index_patterns": [index_pattern],
                 "priority": 501,
                 "template": {
                   "settings": {
                     "index": {
-                       "lifecycle": {"name": "so-" ~ integration_type ~ component_name ~ "-logs"},
+                       "lifecycle": {"name": "so-" ~ integration_type ~ custom_component_name ~ "-logs"},
                         "number_of_replicas": 0
                     }
                   }
@@ -1,101 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-{% from 'allowed_states.map.jinja' import allowed_states %}
-{% if sls in allowed_states %}
-{%   from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %}
-
-include:
-  - elasticfleet.config
-
-# If enabled, automatically update Fleet Logstash Outputs
-{% if ELASTICFLEETMERGED.config.server.enable_auto_configuration and grains.role not in ['so-import', 'so-eval'] %}
-so-elastic-fleet-auto-configure-logstash-outputs:
-  cmd.run:
-    - name: /usr/sbin/so-elastic-fleet-outputs-update
-    - retry:
-        attempts: 4
-        interval: 30
-{% endif %}
-
-# If enabled, automatically update Fleet Server URLs & ES Connection
-so-elastic-fleet-auto-configure-server-urls:
-  cmd.run:
-    - name: /usr/sbin/so-elastic-fleet-urls-update
-    - retry:
-        attempts: 4
-        interval: 30
-
-# Automatically update Fleet Server Elasticsearch URLs & Agent Artifact URLs
-so-elastic-fleet-auto-configure-elasticsearch-urls:
-  cmd.run:
-    - name: /usr/sbin/so-elastic-fleet-es-url-update
-    - retry:
-        attempts: 4
-        interval: 30
-
-so-elastic-fleet-auto-configure-artifact-urls:
-  cmd.run:
-    - name: /usr/sbin/so-elastic-fleet-artifacts-url-update
-    - retry:
-        attempts: 4
-        interval: 30
-
-so-elastic-fleet-package-statefile:
-  file.managed:
-    - name: /opt/so/state/elastic_fleet_packages.txt
-    - contents: {{ELASTICFLEETMERGED.packages}}
-
-so-elastic-fleet-package-upgrade:
-  cmd.run:
-    - name: /usr/sbin/so-elastic-fleet-package-upgrade
-    - retry:
-        attempts: 3
-        interval: 10
-    - onchanges:
-      - file: /opt/so/state/elastic_fleet_packages.txt
-
-so-elastic-fleet-integrations:
-  cmd.run:
-    - name: /usr/sbin/so-elastic-fleet-integration-policy-load
-    - retry:
-        attempts: 3
-        interval: 10
-
-so-elastic-agent-grid-upgrade:
-  cmd.run:
-    - name: /usr/sbin/so-elastic-agent-grid-upgrade
-    - retry:
-        attempts: 12
-        interval: 5
-
-so-elastic-fleet-integration-upgrade:
-  cmd.run:
-    - name: /usr/sbin/so-elastic-fleet-integration-upgrade
-    - retry:
-        attempts: 3
-        interval: 10
-
-{# Optional integrations script doesn't need the retries like so-elastic-fleet-integration-upgrade which loads the default integrations #}
-so-elastic-fleet-addon-integrations:
-  cmd.run:
-    - name: /usr/sbin/so-elastic-fleet-optional-integrations-load
-
-{% if ELASTICFLEETMERGED.config.defend_filters.enable_auto_configuration %}
-so-elastic-defend-manage-filters-file-watch:
-  cmd.run:
-    - name: python3 /sbin/so-elastic-defend-manage-filters.py -c /opt/so/conf/elasticsearch/curl.config -d /opt/so/conf/elastic-fleet/defend-exclusions/disabled-filters.yaml -i /nsm/securityonion-resources/event_filters/ -i /opt/so/conf/elastic-fleet/defend-exclusions/rulesets/custom-filters/ &>> /opt/so/log/elasticfleet/elastic-defend-manage-filters.log
-    - onchanges:
-      - file: elasticdefendcustom
-      - file: elasticdefenddisabled
-{% endif %}
-
-{% else %}
-
-{{sls}}_state_not_allowed:
-  test.fail_without_changes:
-    - name: {{sls}}_state_not_allowed
-
-{% endif %}
@@ -1,15 +1,14 @@
 elasticfleet:
  enabled:
    description: Enables or disables the Elastic Fleet process. This process is critical for managing Elastic Agents.
-    forcedType: bool
    advanced: True
-    helpLink: elastic-fleet
+    helpLink: elastic-fleet.html
  enable_manager_output:
    description: Setting this option to False should only be considered if there is at least one receiver node in the grid. If True, Elastic Agent will send events to the manager and receivers. If False, events will only be send to the receivers.
    advanced: True
    global: True
    forcedType: bool
-    helpLink: elastic-fleet
+    helpLink: elastic-fleet.html
  files:
    soc:
      elastic-defend-disabled-filters__yaml:
@@ -18,7 +17,7 @@ elasticfleet:
        syntax: yaml
        file: True
        global: True
-        helpLink: elastic-fleet
+        helpLink: elastic-fleet.html
        advanced: True
      elastic-defend-custom-filters__yaml:
        title: Custom Elastic Defend filters
@@ -26,32 +25,31 @@ elasticfleet:
        syntax: yaml
        file: True
        global: True
-        helpLink: elastic-fleet
+        helpLink: elastic-fleet.html
        advanced: True
  logging:
    zeek:
      excluded:
        description: This is a list of Zeek logs that are excluded from being shipped through the data processing pipeline. If you remove a log from this list, Elastic Agent will attempt to process it. If an ingest node pipeline is not available to process the logs, you may experience errors.
        forcedType: "[]string"
-        helpLink: zeek
+        helpLink: zeek.html
  config:
    defend_filters:
      enable_auto_configuration:
        description: Enable auto-configuration and management of the Elastic Defend Exclusion filters.
-        forcedType: bool
        global: True
-        helpLink: elastic-fleet
+        helpLink: elastic-fleet.html
        advanced: True
    subscription_integrations:
      description: Enable the installation of integrations that require an Elastic license.
      global: True
      forcedType: bool
-      helpLink: elastic-fleet
+      helpLink: elastic-fleet.html
    auto_upgrade_integrations:
      description: Enables or disables automatically upgrading Elastic Agent integrations.
      global: True
      forcedType: bool
-      helpLink: elastic-fleet
+      helpLink: elastic-fleet.html
    outputs:
      logstash:
        bulk_max_size:
@@ -59,68 +57,67 @@ elasticfleet:
          global: True
          forcedType: int
          advanced: True
-          helpLink: elastic-fleet
+          helpLink: elastic-fleet.html
        worker:
          description: The number of workers per configured host publishing events.
          global: True
          forcedType: int
          advanced: true
-          helpLink: elastic-fleet
+          helpLink: elastic-fleet.html
        queue_mem_events:
          title: queued events
          description: The number of events the queue can store. This value should be evenly divisible by the smaller of 'bulk_max_size' to avoid sending partial batches to the output.
          global: True
          forcedType: int
          advanced: True
-          helpLink: elastic-fleet
+          helpLink: elastic-fleet.html
        timeout:
          description: The number of seconds to wait for responses from the Logstash server before timing out. Eg 30s
          regex: ^[0-9]+s$
          advanced: True
          global: True
-          helpLink: elastic-fleet
+          helpLink: elastic-fleet.html
        loadbalance:
          description: If true and multiple Logstash hosts are configured, the output plugin load balances published events onto all Logstash hosts. If false, the output plugin sends all events to one host (determined at random) and switches to another host if the selected one becomes unresponsive.
          forcedType: bool
          advanced: True
          global: True
-          helpLink: elastic-fleet
+          helpLink: elastic-fleet.html
        compression_level:
          description: The gzip compression level. The compression level must be in the range of 1 (best speed) to 9 (best compression).
          regex: ^[1-9]$
          forcedType: int
          advanced: True
          global: True
-          helpLink: elastic-fleet
+          helpLink: elastic-fleet.html
    server:
      custom_fqdn:
        description: Custom FQDN for Agents to connect to. One per line.
        global: True
-        helpLink: elastic-fleet
+        helpLink: elastic-fleet.html
        advanced: True
        forcedType: "[]string"
      enable_auto_configuration:
        description: Enable auto-configuration of Logstash Outputs & Fleet Host URLs.
-        forcedType: bool
        global: True
-        helpLink: elastic-fleet
+        helpLink: elastic-fleet.html
        advanced: True
      endpoints_enrollment:
        description: Endpoint enrollment key.
        global: True
-        helpLink: elastic-fleet
+        helpLink: elastic-fleet.html
        sensitive: True
        advanced: True
      es_token:
        description: Elastic auth token.
        global: True
-        helpLink: elastic-fleet
+        helpLink: elastic-fleet.html
        sensitive: True
        advanced: True
      grid_enrollment:
        description: Grid enrollment key.
        global: True
-        helpLink: elastic-fleet
+        helpLink: elastic-fleet.html
        sensitive: True
        advanced: True
  optional_integrations:
@@ -128,57 +125,57 @@ elasticfleet:
      enabled_nodes:
        description: Fleet nodes with the Sublime Platform integration enabled. Enter one per line.
        global: True
-        helpLink: elastic-fleet
+        helpLink: elastic-fleet.html
        advanced: True
        forcedType: "[]string"
      api_key:
        description: API key for Sublime Platform.
        global: True
-        helpLink: elastic-fleet
+        helpLink: elastic-fleet.html
        advanced: True
        forcedType: string
        sensitive: True
      base_url:
        description: Base URL for Sublime Platform.
        global: True
-        helpLink: elastic-fleet
+        helpLink: elastic-fleet.html
        advanced: True
        forcedType: string
      poll_interval:
        description: Poll interval for alerts from Sublime Platform.
        global: True
-        helpLink: elastic-fleet
+        helpLink: elastic-fleet.html
        advanced: True
        forcedType: string
      limit:
        description: The maximum number of message groups to return from Sublime Platform.
        global: True
-        helpLink: elastic-fleet
+        helpLink: elastic-fleet.html
        advanced: True
        forcedType: int
    kismet:
      base_url:
        description: Base URL for Kismet.
        global: True
-        helpLink: elastic-fleet
+        helpLink: elastic-fleet.html
        advanced: True
        forcedType: string
      poll_interval:
        description: Poll interval for wireless device data from Kismet. Integration is currently configured to return devices seen as active by any Kismet sensor within the last 10 minutes.
        global: True
-        helpLink: elastic-fleet
+        helpLink: elastic-fleet.html
        advanced: True
        forcedType: string
      api_key:
        description: API key for Kismet.
        global: True
-        helpLink: elastic-fleet
+        helpLink: elastic-fleet.html
        advanced: True
        forcedType: string
        sensitive: True
      enabled_nodes:
        description: Fleet nodes with the Kismet integration enabled. Enter one per line.
        global: True
-        helpLink: elastic-fleet
+        helpLink: elastic-fleet.html
        advanced: True
        forcedType: "[]string"
@@ -135,33 +135,9 @@ elastic_fleet_bulk_package_install() {
    fi
 }

-elastic_fleet_get_package_list_by_type() {
-    if ! output=$(fleet_api "epm/packages"); then
+elastic_fleet_installed_packages() {
+    if ! fleet_api "epm/packages/installed?perPage=500"; then
        return 1
-    else
-        is_integration=$(jq '[.items[] | select(.type=="integration") | .name ]' <<< "$output")
-        is_input=$(jq '[.items[] | select(.type=="input") | .name ]' <<< "$output")
-        is_content=$(jq '[.items[] | select(.type=="content") | .name ]' <<< "$output")
-        jq -n --argjson is_integration "${is_integration:-[]}" \
-              --argjson is_input "${is_input:-[]}" \
-              --argjson is_content "${is_content:-[]}" \
-              '{"integration": $is_integration,"input": $is_input, "content": $is_content}'
-    fi
-}
-elastic_fleet_installed_packages_components() {
-    package_type=${1,,}
-    if [[ "$package_type" != "integration" && "$package_type" != "input" && "$package_type" != "content" ]]; then
-        echo "Error: Invalid package type ${package_type}. Valid types are 'integration', 'input', or 'content'."
-        return 1
-    fi
-
-    packages_by_type=$(elastic_fleet_get_package_list_by_type)
-    packages=$(jq --arg package_type "$package_type" '.[$package_type]' <<< "$packages_by_type")
-
-    if ! output=$(fleet_api "epm/packages/installed?perPage=500"); then
-        return 1
-    else
-        jq -c --argjson packages "$packages" '[.items[] | select(.name | IN($packages[])) | {name: .name, dataStreams: .dataStreams}]' <<< "$output"
    fi
 }

@@ -240,7 +216,7 @@ elastic_fleet_policy_create() {
        --arg DESC "$DESC" \
        --arg TIMEOUT $TIMEOUT \
        --arg FLEETSERVER "$FLEETSERVER" \
-            '{"name": $NAME,"id":$NAME,"description":$DESC,"namespace":"default","monitoring_enabled":["logs"],"inactivity_timeout":$TIMEOUT,"has_fleet_server":$FLEETSERVER,"advanced_settings":{"agent_logging_level": "warning"}}'
+            '{"name": $NAME,"id":$NAME,"description":$DESC,"namespace":"default","monitoring_enabled":["logs"],"inactivity_timeout":$TIMEOUT,"has_fleet_server":$FLEETSERVER}'
        )
    # Create Fleet Policy
    if ! fleet_api "agent_policies" -XPOST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"; then
@@ -5,13 +5,7 @@
 # this file except in compliance with the Elastic License 2.0.

 . /usr/sbin/so-common
-. /usr/sbin/so-elastic-fleet-common
 {%- import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS %}
-{%- import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %}
-{# Optionally override Elasticsearch version for Elastic Agent patch releases #}
-{%- if ELASTICFLEETDEFAULTS.elasticfleet.patch_version is defined %}
-{%-   do ELASTICSEARCHDEFAULTS.elasticsearch.update({'version': ELASTICFLEETDEFAULTS.elasticfleet.patch_version}) %}
-{%- endif %}

 # Only run on Managers
 if ! is_manager_node; then
@@ -20,10 +14,13 @@ if ! is_manager_node; then
 fi

 # Get current list of Grid Node Agents that need to be upgraded
-if ! RAW_JSON=$(fleet_api "agents?perPage=20&page=1&kuery=NOT%20agent.version%3A%20{{ELASTICSEARCHDEFAULTS.elasticsearch.version | urlencode }}%20AND%20policy_id%3A%20so-grid-nodes_%2A&showInactive=false&getStatusSummary=true" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'); then
+RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/agents?perPage=20&page=1&kuery=NOT%20agent.version%3A%20{{ELASTICSEARCHDEFAULTS.elasticsearch.version}}%20AND%20policy_id%3A%20so-grid-nodes_%2A&showInactive=false&getStatusSummary=true" --retry 3 --retry-delay 30 --fail 2>/dev/null)

-    printf "Failed to query for current Grid Agents...\n"
-    exit 1
+# Check to make sure that the server responded with good data - else, bail from script
+CHECKSUM=$(jq -r '.page' <<< "$RAW_JSON")
+if [ "$CHECKSUM" -ne 1 ]; then
+ printf "Failed to query for current Grid Agents...\n"
+ exit 1
 fi

 # Generate list of Node Agents that need updates
@@ -34,12 +31,10 @@ if [ "$OUTDATED_LIST" != '[]' ]; then
   printf "Initiating upgrades for $AGENTNUMBERS Agents to Elastic {{ELASTICSEARCHDEFAULTS.elasticsearch.version}}...\n\n"

   # Generate updated JSON payload
-   JSON_STRING=$(jq -n --arg ELASTICVERSION "{{ELASTICSEARCHDEFAULTS.elasticsearch.version}}" --argjson UPDATELIST "$OUTDATED_LIST" '{"version": $ELASTICVERSION,"agents": $UPDATELIST }')
+   JSON_STRING=$(jq -n --arg ELASTICVERSION {{ELASTICSEARCHDEFAULTS.elasticsearch.version}} --arg UPDATELIST $OUTDATED_LIST '{"version": $ELASTICVERSION,"agents": $UPDATELIST }')

   # Update Node Agents
-   if ! fleet_api "agents/bulk_upgrade" -XPOST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"; then
-       printf "Failed to initiate Agent upgrades...\n"
-   fi
+   curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "http://localhost:5601/api/fleet/agents/bulk_upgrade" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
 else
    printf "No Agents need updates... Exiting\n\n"
    exit 0
@@ -18,9 +18,7 @@ INSTALLED_PACKAGE_LIST=/tmp/esfleet_installed_packages.json
 BULK_INSTALL_PACKAGE_LIST=/tmp/esfleet_bulk_install.json
 BULK_INSTALL_PACKAGE_TMP=/tmp/esfleet_bulk_install_tmp.json
 BULK_INSTALL_OUTPUT=/opt/so/state/esfleet_bulk_install_results.json
-INTEGRATION_PACKAGE_COMPONENTS=/opt/so/state/esfleet_package_components.json
-INPUT_PACKAGE_COMPONENTS=/opt/so/state/esfleet_input_package_components.json
-CONTENT_PACKAGE_COMPONENTS=/opt/so/state/esfleet_content_package_components.json
+PACKAGE_COMPONENTS=/opt/so/state/esfleet_package_components.json
 COMPONENT_TEMPLATES=/opt/so/state/esfleet_component_templates.json

 PENDING_UPDATE=false
@@ -181,13 +179,10 @@ if [[ -f $STATE_FILE_SUCCESS  ]]; then
        else
            echo "Elastic integrations don't appear to need installation/updating..."
        fi
-        # Write out file for generating index/component/ilm templates, keeping each package type separate
-        for package_type in "INTEGRATION" "INPUT" "CONTENT"; do
-            if latest_installed_package_list=$(elastic_fleet_installed_packages_components "$package_type"); then
-                outfile="${package_type}_PACKAGE_COMPONENTS"
-                echo $latest_installed_package_list > "${!outfile}"
-            fi
-        done
+        # Write out file for generating index/component/ilm templates
+        if latest_installed_package_list=$(elastic_fleet_installed_packages); then
+            echo $latest_installed_package_list | jq '[.items[] | {name: .name, es_index_patterns: .dataStreams}]' > $PACKAGE_COMPONENTS
+        fi
        if retry 3 1 "so-elasticsearch-query / --fail --output /dev/null"; then
            # Refresh installed component template list
            latest_component_templates_list=$(so-elasticsearch-query _component_template | jq '.component_templates[] | .name' | jq -s '.')
@@ -235,16 +235,6 @@ function update_kafka_outputs() {

 {% endif %}

-# Compare the current Elastic Fleet certificate against what is on disk
-POLICY_CERT_SHA=$(jq -r '.item.ssl.certificate' <<< $RAW_JSON | openssl x509 -noout -sha256 -fingerprint)
-DISK_CERT_SHA=$(openssl x509 -in /etc/pki/elasticfleet-logstash.crt -noout -sha256 -fingerprint)
-
-if [[ "$POLICY_CERT_SHA" != "$DISK_CERT_SHA" ]]; then
-    printf "Certificate on disk doesn't match certificate in policy - forcing update\n"
-    UPDATE_CERTS=true
-    FORCE_UPDATE=true
-fi
-
 # Sort & hash the new list of Logstash Outputs
 NEW_LIST_JSON=$(jq --compact-output --null-input '$ARGS.positional' --args -- "${NEW_LIST[@]}")
 NEW_HASH=$(sha256sum <<< "$NEW_LIST_JSON" | awk '{print $1}')
@@ -1,164 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-{% from 'allowed_states.map.jinja' import allowed_states %}
-{% if sls in allowed_states %}
-{%   from 'vars/globals.map.jinja' import GLOBALS %}
-{%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
-{%   from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS, SO_MANAGED_INDICES %}
-{%   if GLOBALS.role != 'so-heavynode' %}
-{%     from 'elasticsearch/template.map.jinja' import ALL_ADDON_SETTINGS %}
-{%   endif %}
-
-escomponenttemplates:
-  file.recurse:
-    - name: /opt/so/conf/elasticsearch/templates/component
-    - source: salt://elasticsearch/templates/component
-    - user: 930
-    - group: 939
-    - clean: True
-    - onchanges_in:
-      - file: so-elasticsearch-templates-reload
-    - show_changes: False
-
-# Clean up legacy and non-SO managed templates from the elasticsearch/templates/index/ directory
-so_index_template_dir:
-  file.directory:
-    - name: /opt/so/conf/elasticsearch/templates/index
-    - clean: True
-    {%- if SO_MANAGED_INDICES %}
-    - require:
-      {%- for index in SO_MANAGED_INDICES %}
-      - file: so_index_template_{{index}}
-      {%- endfor %}
-    {%- endif %}
-
-# Auto-generate index templates for SO managed indices (directly defined in elasticsearch/defaults.yaml)
-#   These index templates are for the core SO datasets and are always required
-{%  for index, settings in ES_INDEX_SETTINGS.items() %}
-{%    if settings.index_template is defined %}
-so_index_template_{{index}}:
-  file.managed:
-    - name: /opt/so/conf/elasticsearch/templates/index/{{ index }}-template.json
-    - source: salt://elasticsearch/base-template.json.jinja
-    - defaults:
-        TEMPLATE_CONFIG: {{ settings.index_template }}
-    - template: jinja
-    - onchanges_in:
-      - file: so-elasticsearch-templates-reload
-{%    endif %}
-{%  endfor %}
-
-{%  if GLOBALS.role != "so-heavynode" %}
-# Auto-generate optional index templates for integration | input | content packages
-#   These index templates are not used by default (until user adds package to an agent policy).
-#   Pre-configured with standard defaults, and incorporated into SOC configuration for user customization.
-{%    for index,settings in ALL_ADDON_SETTINGS.items() %}
-{%      if settings.index_template is defined %}
-addon_index_template_{{index}}:
-  file.managed:
-    - name: /opt/so/conf/elasticsearch/templates/addon-index/{{ index }}-template.json
-    - source: salt://elasticsearch/base-template.json.jinja
-    - defaults:
-        TEMPLATE_CONFIG: {{ settings.index_template }}
-    - template: jinja
-    - show_changes: False
-    - onchanges_in:
-      - file: addon-elasticsearch-templates-reload
-{%       endif %}
-{%     endfor %}
-{%   endif %}
-
-{%  if GLOBALS.role in GLOBALS.manager_roles %}
-so-es-cluster-settings:
-  cmd.run:
-    - name: /usr/sbin/so-elasticsearch-cluster-settings
-    - cwd: /opt/so
-    - template: jinja
-    - require:
-      - docker_container: so-elasticsearch
-      - file: elasticsearch_sbin_jinja
-      - http: wait_for_so-elasticsearch
-{%  endif %}
-
-# heavynodes will only load ILM policies for SO managed indices. (Indicies defined in elasticsearch/defaults.yaml)
-so-elasticsearch-ilm-policy-load:
-  cmd.run:
-    - name: /usr/sbin/so-elasticsearch-ilm-policy-load
-    - cwd: /opt/so
-    - require:
-      - docker_container: so-elasticsearch
-      - file: so-elasticsearch-ilm-policy-load-script
-    - onchanges:
-      - file: so-elasticsearch-ilm-policy-load-script
-
-so-elasticsearch-templates-reload:
-  file.absent:
-    - name: /opt/so/state/estemplates.txt
-
-addon-elasticsearch-templates-reload:
-  file.absent:
-    - name: /opt/so/state/addon_estemplates.txt
-
-# so-elasticsearch-templates-load will have its first successful run during the 'so-elastic-fleet-setup' script
-so-elasticsearch-templates:
-  cmd.run:
-{%-  if GLOBALS.role == "so-heavynode" %}
-    - name: /usr/sbin/so-elasticsearch-templates-load --heavynode
-{%-  else %}
-    - name: /usr/sbin/so-elasticsearch-templates-load
-{%-  endif %}
-    - cwd: /opt/so
-    - template: jinja
-    - require:
-      - docker_container: so-elasticsearch
-      - file: elasticsearch_sbin_jinja
-
-so-elasticsearch-pipelines:
-  cmd.run:
-    - name: /usr/sbin/so-elasticsearch-pipelines {{ GLOBALS.hostname }}
-    - require:
-      - docker_container: so-elasticsearch
-      - file: so-elasticsearch-pipelines-script
-
-so-elasticsearch-roles-load:
-  cmd.run:
-    - name: /usr/sbin/so-elasticsearch-roles-load
-    - cwd: /opt/so
-    - template: jinja
-    - require:
-      - docker_container: so-elasticsearch
-      - file: elasticsearch_sbin_jinja
-
-{%  if grains.role in ['so-managersearch', 'so-manager', 'so-managerhype'] %}
-{%    set ap = "absent" %}
-{%  endif %}
-{%  if grains.role in ['so-eval', 'so-standalone', 'so-heavynode'] %}
-{%    if ELASTICSEARCHMERGED.index_clean %}
-{%      set ap = "present" %}
-{%    else %}
-{%      set ap = "absent" %}
-{%    endif %}
-{%  endif %}
-{%  if grains.role in ['so-eval', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-manager'] %}
-so-elasticsearch-indices-delete:
-  cron.{{ap}}:
-    - name: /usr/sbin/so-elasticsearch-indices-delete > /opt/so/log/elasticsearch/cron-elasticsearch-indices-delete.log 2>&1
-    - identifier: so-elasticsearch-indices-delete
-    - user: root
-    - minute: '*/5'
-    - hour: '*'
-    - daymonth: '*'
-    - month: '*'
-    - dayweek: '*'
-{%  endif %}
-
-{% else %}
-
-{{sls}}_state_not_allowed:
-  test.fail_without_changes:
-    - name: {{sls}}_state_not_allowed
-
-{% endif %}
@@ -6,6 +6,8 @@
 {% from 'vars/globals.map.jinja' import GLOBALS %}
 {% import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS with context %}

+{% set HIGHLANDER = salt['pillar.get']('global:highlander', False) %}
+
 {# this is a list of dicts containing hostname:ip for elasticsearch nodes that need to know about each other for cluster #}
 {% set ELASTICSEARCH_SEED_HOSTS = [] %}
 {% set node_data = salt['pillar.get']('elasticsearch:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %}
@@ -34,8 +36,14 @@
      {% endfor %}
    {% endif %}
 {% elif grains.id.split('_') | last == 'searchnode' %}
+  {% if HIGHLANDER %}
+        {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.node.roles.extend(['ml', 'master', 'transform']) %}
+  {% endif %}
  {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.update({'discovery': {'seed_hosts': [GLOBALS.manager]}}) %}
 {% endif %}
+{% if HIGHLANDER %}
+      {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.xpack.ml.update({'enabled': true}) %}
+{% endif %}

 {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.node.update({'name': GLOBALS.hostname}) %}
 {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.cluster.update({'name': GLOBALS.hostname}) %}
@@ -10,7 +10,7 @@

 vm.max_map_count:
  sysctl.present:
-    - value: {{ ELASTICSEARCHMERGED.vm.max_map_count }}
+    - value: 262144

 # Add ES Group
 elasticsearchgroup:
@@ -66,8 +66,6 @@ so-elasticsearch-ilm-policy-load-script:
    - group: 939
    - mode: 754
    - template: jinja
-    - defaults:
-        GLOBALS: {{ GLOBALS }}
    - show_changes: False

 so-elasticsearch-pipelines-script:
@@ -93,13 +91,6 @@ estemplatedir:
    - group: 939
    - makedirs: True

-esaddontemplatedir:
-  file.directory:
-    - name: /opt/so/conf/elasticsearch/templates/addon-index
-    - user: 930
-    - group: 939
-    - makedirs: True
-
 esrolesdir:
  file.directory:
    - name: /opt/so/conf/elasticsearch/roles
@@ -107,6 +98,10 @@ esrolesdir:
    - group: 939
    - makedirs: True

+eslibdir:
+  file.absent:
+    - name: /opt/so/conf/elasticsearch/lib
+
 esingestdynamicconf:
  file.recurse:
    - name: /opt/so/conf/elasticsearch/ingest
@@ -124,6 +119,11 @@ esingestconf:
    - group: 939
    - show_changes: False

+# Remove .fleet_final_pipeline-1 because we are using global@custom now
+so-fleet-final-pipeline-remove:
+  file.absent:
+    - name: /opt/so/conf/elasticsearch/ingest/.fleet_final_pipeline-1
+
 # Auto-generate Elasticsearch ingest node pipelines from pillar
 {% for pipeline, config in ELASTICSEARCHMERGED.pipelines.items() %}
 es_ingest_conf_{{pipeline}}:
@@ -1,9 +1,7 @@
 elasticsearch:
  enabled: false
-  version: 9.3.3
+  version: 9.0.8
  index_clean: true
-  vm:
-    max_map_count: 1048576
  config:
    action:
      destructive_requires_name: true
@@ -119,7 +117,7 @@ elasticsearch:
        ignore_missing_component_templates: []
        index_patterns:
        - so-case*
-        priority: 501
+        priority: 500
        template:
          mappings:
            date_detection: false
@@ -131,6 +129,8 @@ elasticsearch:
                match_mapping_type: string
          settings:
            index:
+              lifecycle:
+                name: so-case-logs
              mapping:
                total_fields:
                  limit: 1500
@@ -141,7 +141,14 @@ elasticsearch:
              sort:
                field: '@timestamp'
                order: desc
+      policy:
+        phases:
+          hot:
+            actions: {}
+            min_age: 0ms
    so-common:
+      close: 30
+      delete: 365
      index_sorting: false
      index_template:
        composed_of:
@@ -205,9 +212,7 @@ elasticsearch:
        - common-settings
        - common-dynamic-mappings
        - winlog-mappings
-        data_stream:
-          allow_custom_routing: false
-          hidden: false
+        data_stream: {}
        ignore_missing_component_templates: []
        index_patterns:
        - logs-*-so*
@@ -267,7 +272,7 @@ elasticsearch:
        ignore_missing_component_templates: []
        index_patterns:
        - so-detection*
-        priority: 501
+        priority: 500
        template:
          mappings:
            date_detection: false
@@ -279,6 +284,8 @@ elasticsearch:
                match_mapping_type: string
          settings:
            index:
+              lifecycle:
+                name: so-detection-logs
              mapping:
                total_fields:
                  limit: 1500
@@ -289,6 +296,11 @@ elasticsearch:
              sort:
                field: '@timestamp'
                order: desc
+      policy:
+        phases:
+          hot:
+            actions: {}
+            min_age: 0ms
    sos-backup:
      index_sorting: false
      index_template:
@@ -448,7 +460,7 @@ elasticsearch:
        ignore_missing_component_templates: []
        index_patterns:
        - endgame*
-        priority: 501
+        priority: 500
        template:
          mappings:
            date_detection: false
@@ -496,6 +508,8 @@ elasticsearch:
                priority: 50
            min_age: 30d
    so-idh:
+      close: 30
+      delete: 365
      index_sorting: false
      index_template:
        composed_of:
@@ -550,13 +564,10 @@ elasticsearch:
        - dtc-user_agent-mappings
        - common-settings
        - common-dynamic-mappings
-        data_stream:
-          allow_custom_routing: false
-          hidden: false
        ignore_missing_component_templates: []
        index_patterns:
-        - logs-idh-so*
-        priority: 501
+        - so-idh-*
+        priority: 500
        template:
          mappings:
            date_detection: false
@@ -666,13 +677,11 @@ elasticsearch:
        - common-dynamic-mappings
        - winlog-mappings
        - hash-mappings
-        data_stream:
-          allow_custom_routing: false
-          hidden: false
+        data_stream: {}
        ignore_missing_component_templates: []
        index_patterns:
        - logs-import-so*
-        priority: 501
+        priority: 500
        template:
          mappings:
            date_detection: false
@@ -727,7 +736,7 @@ elasticsearch:
        ignore_missing_component_templates: []
        index_patterns:
        - so-ip*
-        priority: 501
+        priority: 500
        template:
          mappings:
            date_detection: false
@@ -742,12 +751,19 @@ elasticsearch:
              mapping:
                total_fields:
                  limit: 1500
+              lifecycle:
+                name: so-ip-mappings-logs
              number_of_replicas: 0
              number_of_shards: 1
              refresh_interval: 30s
              sort:
                field: '@timestamp'
                order: desc
+      policy:
+        phases:
+          hot:
+            actions: {}
+            min_age: 0ms
    so-items:
      index_sorting: false
      index_template:
@@ -756,7 +772,7 @@ elasticsearch:
        ignore_missing_component_templates: []
        index_patterns:
        - .items-default-**
-        priority: 501
+        priority: 500
        template:
          mappings:
            date_detection: false
@@ -835,6 +851,8 @@ elasticsearch:
                priority: 50
            min_age: 30d
    so-kratos:
+      close: 30
+      delete: 365
      index_sorting: false
      index_template:
        composed_of:
@@ -855,7 +873,7 @@ elasticsearch:
        ignore_missing_component_templates: []
        index_patterns:
        - logs-kratos-so*
-        priority: 501
+        priority: 500
        template:
          mappings:
            date_detection: false
@@ -903,6 +921,8 @@ elasticsearch:
                priority: 50
            min_age: 30d
    so-hydra:
+      close: 30
+      delete: 365
      index_sorting: false
      index_template:
        composed_of:
@@ -963,7 +983,7 @@ elasticsearch:
        ignore_missing_component_templates: []
        index_patterns:
        - logs-hydra-so*
-        priority: 501
+        priority: 500
        template:
          mappings:
            date_detection: false
@@ -1018,7 +1038,7 @@ elasticsearch:
        ignore_missing_component_templates: []
        index_patterns:
        - .lists-default-**
-        priority: 501
+        priority: 500
        template:
          mappings:
            date_detection: false
@@ -1504,9 +1524,6 @@ elasticsearch:
        - so-fleet_integrations.ip_mappings-1
        - so-fleet_globals-1
        - so-fleet_agent_id_verification-1
-        data_stream:
-          allow_custom_routing: false
-          hidden: false
        ignore_missing_component_templates:
        - logs-elastic_agent.cloudbeat@custom
        index_patterns:
@@ -1742,9 +1759,6 @@ elasticsearch:
        - so-fleet_integrations.ip_mappings-1
        - so-fleet_globals-1
        - so-fleet_agent_id_verification-1
-        data_stream:
-          allow_custom_routing: false
-          hidden: false
        ignore_missing_component_templates:
        - logs-elastic_agent.heartbeat@custom
        index_patterns:
@@ -3004,6 +3018,8 @@ elasticsearch:
                priority: 50
            min_age: 30d
    so-logs-soc:
+      close: 30
+      delete: 365
      index_sorting: false
      index_template:
        composed_of:
@@ -3058,13 +3074,11 @@ elasticsearch:
        - dtc-user_agent-mappings
        - common-settings
        - common-dynamic-mappings
-        data_stream:
-          allow_custom_routing: false
-          hidden: false
+        data_stream: {}
        ignore_missing_component_templates: []
        index_patterns:
        - logs-soc-so*
-        priority: 501
+        priority: 500
        template:
          mappings:
            date_detection: false
@@ -3654,13 +3668,10 @@ elasticsearch:
        - vulnerability-mappings
        - common-settings
        - common-dynamic-mappings
-        data_stream:
-          allow_custom_routing: false
-          hidden: false
        ignore_missing_component_templates: []
        index_patterns:
        - logs-logstash-default*
-        priority: 501
+        priority: 500
        template:
          mappings:
            date_detection: false
@@ -3958,13 +3969,10 @@ elasticsearch:
        - vulnerability-mappings
        - common-settings
        - common-dynamic-mappings
-        data_stream:
-          allow_custom_routing: false
-          hidden: false
        ignore_missing_component_templates: []
        index_patterns:
-        - logs-redis.log*
-        priority: 501
+        - logs-redis-default*
+        priority: 500
        template:
          mappings:
            date_detection: false
@@ -4075,13 +4083,11 @@ elasticsearch:
        - common-settings
        - common-dynamic-mappings
        - hash-mappings
-        data_stream:
-          allow_custom_routing: false
-          hidden: false
+        data_stream: {}
        ignore_missing_component_templates: []
        index_patterns:
        - logs-strelka-so*
-        priority: 501
+        priority: 500
        template:
          mappings:
            date_detection: false
@@ -4191,13 +4197,11 @@ elasticsearch:
        - common-settings
        - common-dynamic-mappings
        - hash-mappings
-        data_stream:
-          allow_custom_routing: false
-          hidden: false
+        data_stream: {}
        ignore_missing_component_templates: []
        index_patterns:
        - logs-suricata-so*
-        priority: 501
+        priority: 500
        template:
          mappings:
            date_detection: false
@@ -4307,13 +4311,11 @@ elasticsearch:
        - common-settings
        - common-dynamic-mappings
        - hash-mappings
-        data_stream:
-          allow_custom_routing: false
-          hidden: false
+        data_stream: {}
        ignore_missing_component_templates: []
        index_patterns:
        - logs-suricata.alerts-*
-        priority: 501
+        priority: 500
        template:
          mappings:
            date_detection: false
@@ -4423,13 +4425,11 @@ elasticsearch:
        - vulnerability-mappings
        - common-settings
        - common-dynamic-mappings
-        data_stream:
-          allow_custom_routing: false
-          hidden: false
+        data_stream: {}
        ignore_missing_component_templates: []
        index_patterns:
        - logs-syslog-so*
-        priority: 501
+        priority: 500
        template:
          mappings:
            date_detection: false
@@ -4541,13 +4541,11 @@ elasticsearch:
        - common-settings
        - common-dynamic-mappings
        - hash-mappings
-        data_stream:
-          allow_custom_routing: false
-          hidden: false
+        data_stream: {}
        ignore_missing_component_templates: []
        index_patterns:
        - logs-zeek-so*
-        priority: 501
+        priority: 500
        template:
          mappings:
            date_detection: false
@@ -6,10 +6,12 @@
 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
-{%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
+{%   from 'docker/docker.map.jinja' import DOCKER %}
 {%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_NODES %}
 {%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_SEED_HOSTS %}
 {%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
+{%   set TEMPLATES = salt['pillar.get']('elasticsearch:templates', {}) %}
+{%   from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS %}

 include:
  - ca
@@ -17,9 +19,6 @@ include:
  - elasticsearch.ssl
  - elasticsearch.config
  - elasticsearch.sostatus
-{%- if GLOBALS.role != "so-searchnode" %}
-  - elasticsearch.cluster
-{%- endif%}

 so-elasticsearch:
  docker_container.running:
@@ -29,15 +28,15 @@ so-elasticsearch:
    - user: elasticsearch
    - networks:
      - sobridge:
-        - ipv4_address: {{ DOCKERMERGED.containers['so-elasticsearch'].ip }}
+        - ipv4_address: {{ DOCKER.containers['so-elasticsearch'].ip }}
    - extra_hosts:
    {% for node in ELASTICSEARCH_NODES %}
    {%   for hostname, ip in node.items() %}
      - {{hostname}}:{{ip}}
    {%   endfor %}
    {% endfor %}
-    {% if DOCKERMERGED.containers['so-elasticsearch'].extra_hosts %}
-      {% for XTRAHOST in DOCKERMERGED.containers['so-elasticsearch'].extra_hosts %}
+    {% if DOCKER.containers['so-elasticsearch'].extra_hosts %}
+      {% for XTRAHOST in DOCKER.containers['so-elasticsearch'].extra_hosts %}
      - {{ XTRAHOST }}
      {% endfor %}
    {% endif %}
@@ -46,19 +45,17 @@ so-elasticsearch:
      - discovery.type=single-node
      {% endif %}
      - ES_JAVA_OPTS=-Xms{{ GLOBALS.elasticsearch.es_heap }} -Xmx{{ GLOBALS.elasticsearch.es_heap }} -Des.transport.cname_in_publish_address=true -Dlog4j2.formatMsgNoLookups=true
-      {% if DOCKERMERGED.containers['so-elasticsearch'].extra_env %}
-        {% for XTRAENV in DOCKERMERGED.containers['so-elasticsearch'].extra_env %}
+      ulimits:
+      - memlock=-1:-1
+      - nofile=65536:65536
+      - nproc=4096
+      {% if DOCKER.containers['so-elasticsearch'].extra_env %}
+        {% for XTRAENV in DOCKER.containers['so-elasticsearch'].extra_env %}
      - {{ XTRAENV }}
        {% endfor %}
      {% endif %}
-    {% if DOCKERMERGED.containers['so-elasticsearch'].ulimits %}
-    - ulimits:
-    {%   for ULIMIT in DOCKERMERGED.containers['so-elasticsearch'].ulimits %}
-      - {{ ULIMIT.name }}={{ ULIMIT.soft }}:{{ ULIMIT.hard }}
-    {%   endfor %}
-    {% endif %}
    - port_bindings:
-      {% for BINDING in DOCKERMERGED.containers['so-elasticsearch'].port_bindings %}
+      {% for BINDING in DOCKER.containers['so-elasticsearch'].port_bindings %}
      - {{ BINDING }}
      {% endfor %}
    - binds:
@@ -78,8 +75,8 @@ so-elasticsearch:
      - {{ repo }}:{{ repo }}:rw
        {% endfor %}
      {% endif %}
-      {% if DOCKERMERGED.containers['so-elasticsearch'].custom_bind_mounts %}
-        {% for BIND in DOCKERMERGED.containers['so-elasticsearch'].custom_bind_mounts %}
+      {% if DOCKER.containers['so-elasticsearch'].custom_bind_mounts %}
+        {% for BIND in DOCKER.containers['so-elasticsearch'].custom_bind_mounts %}
      - {{ BIND }}
        {% endfor %}
      {% endif %}
@@ -102,24 +99,134 @@ so-elasticsearch:
      - cmd: auth_users_roles_inode
      - cmd: auth_users_inode

-wait_for_so-elasticsearch:
-  http.wait_for_successful_query:
-    - name: "https://localhost:9200/"
-    - username: 'so_elastic'
-    - password: '{{ ELASTICSEARCHMERGED.auth.users.so_elastic_user.pass }}'
-    - ssl: True
-    - verify_ssl: False
-    - status: 200
-    - wait_for: 300
-    - request_interval: 15
-    - require:
-      - docker_container: so-elasticsearch
-
 delete_so-elasticsearch_so-status.disabled:
  file.uncomment:
    - name: /opt/so/conf/so-status/so-status.conf
    - regex: ^so-elasticsearch$

+{%   if GLOBALS.role != "so-searchnode" %}
+escomponenttemplates:
+  file.recurse:
+    - name: /opt/so/conf/elasticsearch/templates/component
+    - source: salt://elasticsearch/templates/component
+    - user: 930
+    - group: 939
+    - clean: True
+    - onchanges_in:
+      - file: so-elasticsearch-templates-reload
+    - show_changes: False
+      
+# Auto-generate templates from defaults file
+{%     for index, settings in ES_INDEX_SETTINGS.items() %}
+ {%      if settings.index_template is defined %}
+es_index_template_{{index}}:
+  file.managed:
+    - name: /opt/so/conf/elasticsearch/templates/index/{{ index }}-template.json
+    - source: salt://elasticsearch/base-template.json.jinja
+    - defaults:
+      TEMPLATE_CONFIG: {{ settings.index_template }}
+    - template: jinja
+    - show_changes: False
+    - onchanges_in:
+      - file: so-elasticsearch-templates-reload
+{%       endif %}
+{%     endfor %}
+
+{%     if TEMPLATES %}
+# Sync custom templates to /opt/so/conf/elasticsearch/templates
+{%       for TEMPLATE in TEMPLATES %}
+es_template_{{TEMPLATE.split('.')[0] | replace("/","_") }}:
+  file.managed:
+    - source: salt://elasticsearch/templates/index/{{TEMPLATE}}
+{%         if 'jinja' in TEMPLATE.split('.')[-1] %}
+    - name: /opt/so/conf/elasticsearch/templates/index/{{TEMPLATE.split('/')[1] | replace(".jinja", "")}}
+    - template: jinja
+{%         else %}
+    - name: /opt/so/conf/elasticsearch/templates/index/{{TEMPLATE.split('/')[1]}}
+{%         endif %}
+    - user: 930
+    - group: 939
+    - show_changes: False
+    - onchanges_in:
+      - file: so-elasticsearch-templates-reload
+{%       endfor %}
+{%     endif %}
+
+{%     if GLOBALS.role in GLOBALS.manager_roles %}
+so-es-cluster-settings:
+  cmd.run:
+    - name: /usr/sbin/so-elasticsearch-cluster-settings
+    - cwd: /opt/so
+    - template: jinja
+    - require:
+      - docker_container: so-elasticsearch
+      - file: elasticsearch_sbin_jinja
+{%     endif %}
+
+so-elasticsearch-ilm-policy-load:
+  cmd.run:
+    - name: /usr/sbin/so-elasticsearch-ilm-policy-load
+    - cwd: /opt/so
+    - require:
+      - docker_container: so-elasticsearch
+      - file: so-elasticsearch-ilm-policy-load-script
+    - onchanges:
+      - file: so-elasticsearch-ilm-policy-load-script
+
+so-elasticsearch-templates-reload:
+  file.absent:
+    - name: /opt/so/state/estemplates.txt
+
+so-elasticsearch-templates:
+  cmd.run:
+    - name: /usr/sbin/so-elasticsearch-templates-load
+    - cwd: /opt/so
+    - template: jinja
+    - require:
+      - docker_container: so-elasticsearch
+      - file: elasticsearch_sbin_jinja
+
+so-elasticsearch-pipelines:
+  cmd.run:
+    - name: /usr/sbin/so-elasticsearch-pipelines {{ GLOBALS.hostname }}
+    - require:
+      - docker_container: so-elasticsearch
+      - file: so-elasticsearch-pipelines-script
+
+so-elasticsearch-roles-load:
+  cmd.run:
+    - name: /usr/sbin/so-elasticsearch-roles-load
+    - cwd: /opt/so
+    - template: jinja
+    - require:
+      - docker_container: so-elasticsearch
+      - file: elasticsearch_sbin_jinja
+
+{%     if grains.role in ['so-managersearch', 'so-manager', 'so-managerhype'] %}
+{%       set ap = "absent" %}
+{%     endif %}
+{%     if grains.role in ['so-eval', 'so-standalone', 'so-heavynode'] %}
+{%       if ELASTICSEARCHMERGED.index_clean %}
+{%         set ap = "present" %}
+{%       else %}
+{%         set ap = "absent" %}
+{%       endif %}
+{%     endif %}  
+{%     if grains.role in ['so-eval', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-manager'] %}
+so-elasticsearch-indices-delete:
+  cron.{{ap}}:
+    - name: /usr/sbin/so-elasticsearch-indices-delete > /opt/so/log/elasticsearch/cron-elasticsearch-indices-delete.log 2>&1
+    - identifier: so-elasticsearch-indices-delete
+    - user: root
+    - minute: '*/5'
+    - hour: '*'
+    - daymonth: '*'
+    - month: '*'
+    - dayweek: '*'
+{%     endif %}
+
+{%   endif %}
+
 {% else %}

 {{sls}}_state_not_allowed:
@@ -1,3 +1,5 @@
+{%- set HIGHLANDER = salt['pillar.get']('global:highlander', False) -%}
+{%- raw -%}
 {
  "description" : "common",
  "processors" : [
@@ -65,7 +67,19 @@
    { "append":          { "if": "ctx.dataset_tag_temp != null", "field": "tags", "value": "{{dataset_tag_temp.1}}"  } },
    { "grok":            { "if": "ctx.http?.response?.status_code != null", "field": "http.response.status_code", "patterns": ["%{NUMBER:http.response.status_code:long} %{GREEDYDATA}"]} },
    { "set":             { "if": "ctx?.metadata?.kafka != null" , "field": "kafka.id", "value": "{{metadata.kafka.partition}}{{metadata.kafka.offset}}{{metadata.kafka.timestamp}}", "ignore_failure": true } },
-    { "remove":          { "field": [ "message2", "type", "fields", "category", "module", "dataset", "dataset_tag_temp", "event.dataset_temp" ], "ignore_missing": true, "ignore_failure": true } },
+    { "remove":          { "field": [ "message2", "type", "fields", "category", "module", "dataset", "dataset_tag_temp", "event.dataset_temp" ], "ignore_missing": true, "ignore_failure": true } }
+{%- endraw %}
+{%- if HIGHLANDER %}
+    ,
+    {
+      "pipeline": {
+        "name": "ecs"
+      }
+    }
+{%- endif %}
+{%- raw %}
+    ,
    { "pipeline": { "name": "global@custom", "ignore_missing_pipeline": true, "description": "[Fleet] Global pipeline for all data streams" } }
  ]
 }
+{% endraw %}
@@ -10,28 +10,24 @@
  "processors": [
    {
      "set": {
-        "tag": "set_ecs_version_f5923549",
        "field": "ecs.version",
        "value": "8.17.0"
      }
    },
    {
      "set": {
-        "tag": "set_observer_vendor_ad9d35cc",
        "field": "observer.vendor",
        "value": "netgate"
      }
    },
    {
      "set": {
-        "tag": "set_observer_type_5dddf3ba",
        "field": "observer.type",
        "value": "firewall"
      }
    },
    {
      "rename": {
-        "tag": "rename_message_to_event_original_56a77271",
        "field": "message",
        "target_field": "event.original",
        "ignore_missing": true,
@@ -40,14 +36,12 @@
    },
    {
      "set": {
-        "tag": "set_event_kind_de80643c",
        "field": "event.kind",
        "value": "event"
      }
    },
    {
      "set": {
-        "tag": "set_event_timezone_4ca44cac",
        "field": "event.timezone",
        "value": "{{{_tmp.tz_offset}}}",
        "if": "ctx._tmp?.tz_offset != null && ctx._tmp?.tz_offset != 'local'"
@@ -55,7 +49,6 @@
    },
    {
      "grok": {
-        "tag": "grok_event_original_27d9c8c7",
        "description": "Parse syslog header",
        "field": "event.original",
        "patterns": [
@@ -79,7 +72,6 @@
    },
    {
      "date": {
-        "tag": "date__tmp_timestamp8601_to_timestamp_6ac9d3ce",
        "if": "ctx._tmp.timestamp8601 != null",
        "field": "_tmp.timestamp8601",
        "target_field": "@timestamp",
@@ -90,7 +82,6 @@
    },
    {
      "date": {
-        "tag": "date__tmp_timestamp_to_timestamp_f21e536e",
        "if": "ctx.event?.timezone != null && ctx._tmp?.timestamp != null",
        "field": "_tmp.timestamp",
        "target_field": "@timestamp",
@@ -104,7 +95,6 @@
    },
    {
      "grok": {
-        "tag": "grok_process_name_cef3d489",
        "description": "Set Event Provider",
        "field": "process.name",
        "patterns": [
@@ -117,83 +107,71 @@
    },
    {
      "pipeline": {
-        "tag": "pipeline_e16851a7",
-        "name": "logs-pfsense.log-1.25.2-firewall",
+        "name": "logs-pfsense.log-1.23.1-firewall",
        "if": "ctx.event.provider == 'filterlog'"
      }
    },
    {
      "pipeline": {
-        "tag": "pipeline_828590b5",
-        "name": "logs-pfsense.log-1.25.2-openvpn",
+        "name": "logs-pfsense.log-1.23.1-openvpn",
        "if": "ctx.event.provider == 'openvpn'"
      }
    },
    {
      "pipeline": {
-        "tag": "pipeline_9d37039c",
-        "name": "logs-pfsense.log-1.25.2-ipsec",
+        "name": "logs-pfsense.log-1.23.1-ipsec",
        "if": "ctx.event.provider == 'charon'"
      }
    },
    {
      "pipeline": {
-        "tag": "pipeline_ad56bbca",
-        "name": "logs-pfsense.log-1.25.2-dhcp",
-        "if": "[\"dhcpd\", \"dhclient\", \"dhcp6c\", \"dnsmasq-dhcp\"].contains(ctx.event.provider)"
+        "name": "logs-pfsense.log-1.23.1-dhcp",
+        "if": "[\"dhcpd\", \"dhclient\", \"dhcp6c\"].contains(ctx.event.provider)"
      }
    },
    {
      "pipeline": {
-        "tag": "pipeline_dd85553d",
-        "name": "logs-pfsense.log-1.25.2-unbound",
+        "name": "logs-pfsense.log-1.23.1-unbound",
        "if": "ctx.event.provider == 'unbound'"
      }
    },
    {
      "pipeline": {
-        "tag": "pipeline_720ed255",
-        "name": "logs-pfsense.log-1.25.2-haproxy",
+        "name": "logs-pfsense.log-1.23.1-haproxy",
        "if": "ctx.event.provider == 'haproxy'"
      }
    },
    {
      "pipeline": {
-        "tag": "pipeline_456beba5",
-        "name": "logs-pfsense.log-1.25.2-php-fpm",
+        "name": "logs-pfsense.log-1.23.1-php-fpm",
        "if": "ctx.event.provider == 'php-fpm'"
      }
    },
    {
      "pipeline": {
-        "tag": "pipeline_a0d89375",
-        "name": "logs-pfsense.log-1.25.2-squid",
+        "name": "logs-pfsense.log-1.23.1-squid",
        "if": "ctx.event.provider == 'squid'"
      }
    },
    {
      "pipeline": {
-        "tag": "pipeline_c2f1ed55",
-        "name": "logs-pfsense.log-1.25.2-snort",
+        "name": "logs-pfsense.log-1.23.1-snort",
        "if": "ctx.event.provider == 'snort'"
      }
    },
    {
      "pipeline": {
-        "tag":"pipeline_33db1c9e",
-        "name": "logs-pfsense.log-1.25.2-suricata",
+        "name": "logs-pfsense.log-1.23.1-suricata",
        "if": "ctx.event.provider == 'suricata'"
      }
    },
    {
      "drop": {
-        "tag": "drop_9d7c46f8",
-        "if": "![\"filterlog\", \"openvpn\", \"charon\", \"dhcpd\", \"dnsmasq-dhcp\", \"dhclient\", \"dhcp6c\", \"unbound\", \"haproxy\", \"php-fpm\", \"squid\", \"snort\", \"suricata\"].contains(ctx.event?.provider)"
+        "if": "![\"filterlog\", \"openvpn\", \"charon\", \"dhcpd\", \"dhclient\", \"dhcp6c\", \"unbound\", \"haproxy\", \"php-fpm\", \"squid\", \"snort\", \"suricata\"].contains(ctx.event?.provider)"
      }
    },
    {
      "append": {
-        "tag": "append_event_category_4780a983",
        "field": "event.category",
        "value": "network",
        "if": "ctx.network != null"
@@ -201,7 +179,6 @@
    },
    {
      "convert": {
-        "tag": "convert_source_address_to_source_ip_f5632a20",
        "field": "source.address",
        "target_field": "source.ip",
        "type": "ip",
@@ -211,7 +188,6 @@
    },
    {
      "convert": {
-        "tag": "convert_destination_address_to_destination_ip_f1388f0c",
        "field": "destination.address",
        "target_field": "destination.ip",
        "type": "ip",
@@ -221,7 +197,6 @@
    },
    {
      "set": {
-        "tag": "set_network_type_1f1d940a",
        "field": "network.type",
        "value": "ipv6",
        "if": "ctx.source?.ip != null && ctx.source.ip.contains(\":\")"
@@ -229,7 +204,6 @@
    },
    {
      "set": {
-        "tag": "set_network_type_69deca38",
        "field": "network.type",
        "value": "ipv4",
        "if": "ctx.source?.ip != null && ctx.source.ip.contains(\".\")"
@@ -237,7 +211,6 @@
    },
    {
      "geoip": {
-        "tag": "geoip_source_ip_to_source_geo_da2e41b2",
        "field": "source.ip",
        "target_field": "source.geo",
        "ignore_missing": true
@@ -245,7 +218,6 @@
    },
    {
      "geoip": {
-        "tag": "geoip_destination_ip_to_destination_geo_ab5e2968",
        "field": "destination.ip",
        "target_field": "destination.geo",
        "ignore_missing": true
@@ -253,7 +225,6 @@
    },
    {
      "geoip": {
-        "tag": "geoip_source_ip_to_source_as_28d69883",
        "ignore_missing": true,
        "database_file": "GeoLite2-ASN.mmdb",
        "field": "source.ip",
@@ -266,7 +237,6 @@
    },
    {
      "geoip": {
-        "tag": "geoip_destination_ip_to_destination_as_8a007787",
        "database_file": "GeoLite2-ASN.mmdb",
        "field": "destination.ip",
        "target_field": "destination.as",
@@ -279,7 +249,6 @@
    },
    {
      "rename": {
-        "tag": "rename_source_as_asn_to_source_as_number_a917047d",
        "field": "source.as.asn",
        "target_field": "source.as.number",
        "ignore_missing": true
@@ -287,7 +256,6 @@
    },
    {
      "rename": {
-        "tag": "rename_source_as_organization_name_to_source_as_organization_name_f1362d0b",
        "field": "source.as.organization_name",
        "target_field": "source.as.organization.name",
        "ignore_missing": true
@@ -295,7 +263,6 @@
    },
    {
      "rename": {
-        "tag": "rename_destination_as_asn_to_destination_as_number_3b459fcd",
        "field": "destination.as.asn",
        "target_field": "destination.as.number",
        "ignore_missing": true
@@ -303,7 +270,6 @@
    },
    {
      "rename": {
-        "tag": "rename_destination_as_organization_name_to_destination_as_organization_name_814bd459",
        "field": "destination.as.organization_name",
        "target_field": "destination.as.organization.name",
        "ignore_missing": true
@@ -311,14 +277,12 @@
    },
    {
      "community_id": {
-        "tag": "community_id_d2308e7a",
        "target_field": "network.community_id",
        "ignore_failure": true
      }
    },
    {
      "grok": {
-        "tag": "grok_observer_ingress_interface_name_968018d3",
        "field": "observer.ingress.interface.name",
        "patterns": [
          "%{DATA}.%{NONNEGINT:observer.ingress.vlan.id}"
@@ -329,7 +293,6 @@
    },
    {
      "set": {
-        "tag": "set_network_vlan_id_efd4d96a",
        "field": "network.vlan.id",
        "copy_from": "observer.ingress.vlan.id",
        "ignore_empty_value": true
@@ -337,7 +300,6 @@
    },
    {
      "append": {
-        "tag": "append_related_ip_c1a6356b",
        "field": "related.ip",
        "value": "{{{destination.ip}}}",
        "allow_duplicates": false,
@@ -346,7 +308,6 @@
    },
    {
      "append": {
-        "tag": "append_related_ip_8121c591",
        "field": "related.ip",
        "value": "{{{source.ip}}}",
        "allow_duplicates": false,
@@ -355,7 +316,6 @@
    },
    {
      "append": {
-        "tag": "append_related_ip_53b62ed8",
        "field": "related.ip",
        "value": "{{{source.nat.ip}}}",
        "allow_duplicates": false,
@@ -364,7 +324,6 @@
    },
    {
      "append": {
-        "tag": "append_related_hosts_6f162628",
        "field": "related.hosts",
        "value": "{{{destination.domain}}}",
        "if": "ctx.destination?.domain != null"
@@ -372,7 +331,6 @@
    },
    {
      "append": {
-        "tag": "append_related_user_c036eec2",
        "field": "related.user",
        "value": "{{{user.name}}}",
        "if": "ctx.user?.name != null"
@@ -380,7 +338,6 @@
    },
    {
      "set": {
-        "tag": "set_network_direction_cb1e3125",
        "field": "network.direction",
        "value": "{{{network.direction}}}bound",
        "if": "ctx.network?.direction != null && ctx.network?.direction =~ /^(in|out)$/"
@@ -388,7 +345,6 @@
    },
    {
      "remove": {
-        "tag": "remove_a82e20f2",
        "field": [
          "_tmp"
        ],
@@ -397,21 +353,11 @@
    },
    {
      "script": {
-        "tag": "script_a7f2c062",
        "lang": "painless",
        "description": "This script processor iterates over the whole document to remove fields with null values.",
        "source": "void handleMap(Map map) {\n  for (def x : map.values()) {\n    if (x instanceof Map) {\n        handleMap(x);\n    } else if (x instanceof List) {\n        handleList(x);\n    }\n  }\n  map.values().removeIf(v -> v == null || (v instanceof String && v == \"-\"));\n}\nvoid handleList(List list) {\n  for (def x : list) {\n      if (x instanceof Map) {\n          handleMap(x);\n      } else if (x instanceof List) {\n          handleList(x);\n      }\n  }\n}\nhandleMap(ctx);\n"
      }
    },
-    {
-      "append": {
-        "tag": "append_preserve_original_event_on_error",
-        "field": "tags",
-        "value": "preserve_original_event",
-        "allow_duplicates": false,
-        "if": "ctx.error?.message != null"
-      }
-    },
    {
      "pipeline": {
        "name": "global@custom",
@@ -459,14 +405,7 @@
    {
      "append": {
        "field": "error.message",
-        "value": "Processor '{{{ _ingest.on_failure_processor_type }}}' {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}'"
-      }
-    },
-    {
-      "append": {
-        "field": "tags",
-        "value": "preserve_original_event",
-        "allow_duplicates": false
+        "value": "{{{ _ingest.on_failure_message }}}"
      }
    }
  ]
@@ -22,12 +22,6 @@
                "ignore_failure": true
            }
        },
-        {
-            "lowercase": {
-                "field": "network.transport",
-                "ignore_failure": true
-            }
-        },
        {
            "rename": {
                "field": "message2.in_iface",
@@ -1,18 +0,0 @@
-{
-  "description" : "zeek.websocket",
-  "processors" : [
-    { "set":      { "field": "event.dataset",                           "value": "websocket" } },
-    { "remove":   { "field": ["host"],                                                          "ignore_failure": true    } },
-    { "json":     { "field": "message",                                 "target_field": "message2",                       "ignore_failure": true    } },
-    { "rename": 	{ "field": "message2.host",	                    "target_field": "websocket.host",		              "ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.uri",	                    "target_field": "websocket.uri",		              "ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.user_agent",	            "target_field": "websocket.user_agent",		          "ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.subprotocol",	            "target_field": "websocket.subprotocol",		      "ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.client_protocols",	    "target_field": "websocket.client_protocols",	      "ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.client_extensions",	    "target_field": "websocket.client_extensions",	      "ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.server_extensions",	    "target_field": "websocket.server_extensions",	      "ignore_missing": true 	} },
-    { "remove":		{ "field": "message2.tags",		                                                                      "ignore_failure": true    } },
-    { "set":      { "field": "network.transport",                       "value": "tcp"  } },
-    { "pipeline":       { "name": "zeek.common" } }
-  ]
-}
@@ -45,7 +45,3 @@ appender.rolling_json.strategy.action.condition.nested_condition.age = 1D
 rootLogger.level = info
 rootLogger.appenderRef.rolling.ref = rolling
 rootLogger.appenderRef.rolling_json.ref = rolling_json
-
-# Suppress NotEntitledException WARNs (ES 9.3.3 bug)
-logger.entitlement_security.name = org.elasticsearch.entitlement.runtime.policy.PolicyManager.x-pack-security.org.elasticsearch.security.org.elasticsearch.xpack.security
-logger.entitlement_security.level = error
@@ -1,9 +1,8 @@
 elasticsearch:
  enabled:
    description: Enables or disables the Elasticsearch process. This process provides the log event storage system. WARNING - Disabling this process is unsupported.
-    forcedType: bool
    advanced: True
-    helpLink: elasticsearch
+    helpLink: elasticsearch.html
  version:
    description: "This specifies the version of the following containers: so-elastic-fleet-package-registry, so-elastic-agent, so-elastic-fleet, so-kibana, so-logstash and so-elasticsearch. Modifying this value in the Elasticsearch defaults.yaml will result in catastrophic grid failure."
    readonly: True
@@ -11,20 +10,15 @@ elasticsearch:
    advanced: True
  esheap:
    description: Specify the memory heap size in (m)egabytes for Elasticsearch.
-    helpLink: elasticsearch
+    helpLink: elasticsearch.html
  index_clean:
    description: Determines if indices should be considered for deletion by available disk space in the cluster. Otherwise, indices will only be deleted by the age defined in the ILM settings. This setting only applies to EVAL, STANDALONE, and HEAVY NODE installations. Other installations can only use ILM settings.
    forcedType: bool
-    helpLink: elasticsearch
-  vm:
-    max_map_count:
-      description: The maximum number of memory map areas a process may use. Elasticsearch uses a mmapfs directory by default to store its indices. The default operating system limits on mmap counts could be too low, which may result in out of memory exceptions.
-      forcedType: int
-      helpLink: elasticsearch
+    helpLink: elasticsearch.html
  retention: 
    retention_pct:
      decription: Total percentage of space used by Elasticsearch for multi node clusters
-      helpLink: elasticsearch
+      helpLink: elasticsearch.html
      global: True
  config:
    cluster:
@@ -32,102 +26,55 @@ elasticsearch:
        description: The name of the Security Onion Elasticsearch cluster, for identification purposes.
        readonly: True
        global: True
-        helpLink: elasticsearch
+        helpLink: elasticsearch.html
      logsdb:
        enabled:
          description: Enables or disables the Elasticsearch logsdb index mode. When enabled, most logs-* datastreams will convert to logsdb from standard after rolling over.
          forcedType: bool
          global: True
          advanced: True
-          helpLink: elasticsearch
+          helpLink: elasticsearch.html
      routing:
        allocation:
          disk:
-            threshold_enabled:
+            threshold_enabled: 
              description: Specifies whether the Elasticsearch node will monitor the available disk space for low disk space conditions and take action to protect the cluster.
-              forcedType: bool
-              helpLink: elasticsearch
+              helpLink: elasticsearch.html
            watermark:
              low: 
                description: The lower percentage of used disk space representing a healthy node.
-                helpLink: elasticsearch
+                helpLink: elasticsearch.html
              high: 
                description: The higher percentage of used disk space representing an unhealthy node.
-                helpLink: elasticsearch
+                helpLink: elasticsearch.html
              flood_stage: 
                description: The max percentage of used disk space that will cause the node to take protective actions, such as blocking incoming events.
-                helpLink: elasticsearch
-    action:
-      destructive_requires_name:
-        description: Requires explicit index names when deleting indices. Prevents accidental deletion of indices via wildcard patterns.
-        advanced: True
-        forcedType: bool
-        helpLink: elasticsearch
+                helpLink: elasticsearch.html
    script:
-      max_compilations_rate:
+      max_compilations_rate: 
        description: Max rate of script compilations permitted in the Elasticsearch cluster. Larger values will consume more resources.
        global: True
-        helpLink: elasticsearch
+        helpLink: elasticsearch.html
    indices:
-      id_field_data:
-        enabled:
-          description: Enables or disables loading of field data on the _id field.
-          advanced: True
-          forcedType: bool
-          helpLink: elasticsearch
      query:
        bool:
-          max_clause_count:
+          max_clause_count: 
            description: Max number of boolean clauses per query.
            global: True
-            helpLink: elasticsearch
-    xpack:
-      ml:
-        enabled:
-          description: Enables or disables machine learning on the node.
-          forcedType: bool
-          advanced: True
-          helpLink: elasticsearch
-      security:
-        enabled:
-          description: Enables or disables Elasticsearch security features.
-          forcedType: bool
-          advanced: True
-          helpLink: elasticsearch
-        authc:
-          anonymous:
-            authz_exception:
-              description: Controls whether an authorization exception is thrown when anonymous user does not have the required privileges.
-              advanced: True
-              forcedType: bool
-              helpLink: elasticsearch
-        http:
-          ssl:
-            enabled:
-              description: Enables or disables TLS/SSL for the HTTP layer.
-              advanced: True
-              forcedType: bool
-              helpLink: elasticsearch
-        transport:
-          ssl:
-            enabled:
-              description: Enables or disables TLS/SSL for the transport layer.
-              advanced: True
-              forcedType: bool
-              helpLink: elasticsearch
+            helpLink: elasticsearch.html
  pipelines:
    custom001: &pipelines
      description:
        description: Description of the ingest node pipeline
        global: True
        advanced: True
-        helpLink: elasticsearch
+        helpLink: elasticsearch.html
      processors:
        description: Processors for the ingest node pipeline
        global: True
        advanced: True
        multiline: True
-        helpLink: elasticsearch
+        helpLink: elasticsearch.html
    custom002: *pipelines
    custom003: *pipelines
    custom004: *pipelines
@@ -147,24 +94,24 @@ elasticsearch:
                description: Number of replicas required for all indices. Multiple replicas protects against data loss, but also increases storage costs. This setting will be applied to all indices.
                forcedType: int
                global: True
-                helpLink: elasticsearch
+                helpLink: elasticsearch.html
              refresh_interval: 
                description: Seconds between index refreshes. Shorter intervals can cause query performance to suffer since this is a synchronous and resource-intensive operation.
                global: True
-                helpLink: elasticsearch
+                helpLink: elasticsearch.html
              number_of_shards: 
                description: Number of shards required for this index. Using multiple shards increases fault tolerance, but also increases storage and network costs.
                global: True
-                helpLink: elasticsearch
+                helpLink: elasticsearch.html
              sort:
                field:
                  description: The field to sort by. Must set index_sorting to True.
                  global: True
-                  helpLink: elasticsearch
+                  helpLink: elasticsearch.html
                order:
                  description: The order to sort by. Must set index_sorting to True.
                  global: True
-                  helpLink: elasticsearch
+                  helpLink: elasticsearch.html
      policy:
        phases:
          hot:
@@ -174,16 +121,16 @@ elasticsearch:
                  description: Priority of index. This is used for recovery after a node restart. Indices with higher priorities are recovered before indices with lower priorities.
                  forcedType: int
                  global: True
-                  helpLink: elasticsearch
+                  helpLink: elasticsearch.html
              rollover:
                max_age:
                  description: Maximum age of index.  Once an index reaches this limit, it will be rolled over into a new index.
                  global: True
-                  helpLink: elasticsearch
+                  helpLink: elasticsearch.html
                max_primary_shard_size:
                  description: Maximum primary shard size. Once an index reaches this limit, it will be rolled over into a new index.
                  global: True
-                  helpLink: elasticsearch
+                  helpLink: elasticsearch.html
              shrink:
                method:
                  description: Shrink the index to a new index with fewer primary shards. Shrink operation is by count or size.
@@ -231,13 +178,13 @@ elasticsearch:
              regex: ^[0-9]{1,5}d$
              forcedType: string
              global: True
-              helpLink: elasticsearch
+              helpLink: elasticsearch.html
            actions:
              set_priority:
                priority:
                  description: Used for index recovery after a node restart. Indices with higher priorities are recovered before indices with lower priorities.
                  global: True
-                  helpLink: elasticsearch
+                  helpLink: elasticsearch.html
              allocate:
                number_of_replicas:
                  description: Set the number of replicas. Remains the same as the previous phase by default.
@@ -250,14 +197,14 @@ elasticsearch:
              regex: ^[0-9]{1,5}d$
              forcedType: string
              global: True
-              helpLink: elasticsearch
+              helpLink: elasticsearch.html
            actions:
              set_priority:
                priority:
                  description: Priority of index. This is used for recovery after a node restart. Indices with higher priorities are recovered before indices with lower priorities.
                  forcedType: int
                  global: True
-                  helpLink: elasticsearch
+                  helpLink: elasticsearch.html
              shrink:
                method:
                  description: Shrink the index to a new index with fewer primary shards. Shrink operation is by count or size.
@@ -310,14 +257,13 @@ elasticsearch:
              regex: ^[0-9]{1,5}d$
              forcedType: string
              global: True
-              helpLink: elasticsearch
+              helpLink: elasticsearch.html
    so-logs: &indexSettings
-      index_sorting:
+      index_sorting: 
        description: Sorts the index by event time, at the cost of additional processing resource consumption.
-        forcedType: bool
        global: True
        advanced: True
-        helpLink: elasticsearch
+        helpLink: elasticsearch.html
      index_template:
        index_patterns:
          description: Patterns for matching multiple indices or tables.
@@ -325,7 +271,7 @@ elasticsearch:
          multiline: True
          global: True
          advanced: True
-          helpLink: elasticsearch
+          helpLink: elasticsearch.html
        template:
          settings:
            index:
@@ -334,35 +280,35 @@ elasticsearch:
                forcedType: int
                global: True
                advanced: True
-                helpLink: elasticsearch
+                helpLink: elasticsearch.html
              mapping:
                total_fields:
                  limit:
                    description: Max number of fields that can exist on a single index. Larger values will consume more resources.
                    global: True
                    advanced: True
-                    helpLink: elasticsearch
+                    helpLink: elasticsearch.html
              refresh_interval: 
                description: Seconds between index refreshes. Shorter intervals can cause query performance to suffer since this is a synchronous and resource-intensive operation.
                global: True
                advanced: True
-                helpLink: elasticsearch
+                helpLink: elasticsearch.html
              number_of_shards: 
                description: Number of shards required for this index. Using multiple shards increases fault tolerance, but also increases storage and network costs.
                global: True
                advanced: True
-                helpLink: elasticsearch
+                helpLink: elasticsearch.html
              sort:
                field:
                  description: The field to sort by. Must set index_sorting to True.
                  global: True
                  advanced: True
-                  helpLink: elasticsearch
+                  helpLink: elasticsearch.html
                order:
                  description: The order to sort by. Must set index_sorting to True.
                  global: True
                  advanced: True
-                  helpLink: elasticsearch
+                  helpLink: elasticsearch.html
          mappings:
            _meta:
              package:
@@ -370,43 +316,43 @@ elasticsearch:
                  description: Meta settings for the mapping.
                  global: True
                  advanced: True
-                  helpLink: elasticsearch
+                  helpLink: elasticsearch.html
              managed_by:
                  description: Meta settings for the mapping.
                  global: True
                  advanced: True
-                  helpLink: elasticsearch
+                  helpLink: elasticsearch.html
              managed:
                  description: Meta settings for the mapping.
                  forcedType: bool
                  global: True
                  advanced: True
-                  helpLink: elasticsearch
+                  helpLink: elasticsearch.html
        composed_of:
          description: The index template is composed of these component templates.
          forcedType: "[]string"
          global: True
          advanced: True
-          helpLink: elasticsearch
+          helpLink: elasticsearch.html
        priority:
          description: The priority of the index template.
          forcedType: int
          global: True
          advanced: True
-          helpLink: elasticsearch
+          helpLink: elasticsearch.html
        data_stream:
          hidden:
            description: Hide the data stream.
            forcedType: bool
            global: True
            advanced: True
-            helpLink: elasticsearch
+            helpLink: elasticsearch.html
          allow_custom_routing:
            description: Allow custom routing for the data stream.
            forcedType: bool
            global: True
            advanced: True
-            helpLink: elasticsearch
+            helpLink: elasticsearch.html
      policy:
        phases:
          hot:
@@ -414,7 +360,7 @@ elasticsearch:
              description: Minimum age of index. This determines when the index should be moved to the hot tier.
              global: True
              advanced: True
-              helpLink: elasticsearch
+              helpLink: elasticsearch.html
            actions:
              set_priority:
                priority:
@@ -422,18 +368,18 @@ elasticsearch:
                  forcedType: int
                  global: True
                  advanced: True
-                  helpLink: elasticsearch
+                  helpLink: elasticsearch.html
              rollover:
                max_age:
                  description: Maximum age of index.  Once an index reaches this limit, it will be rolled over into a new index.
                  global: True
                  advanced: True
-                  helpLink: elasticsearch
+                  helpLink: elasticsearch.html
                max_primary_shard_size:
                  description: Maximum primary shard size. Once an index reaches this limit, it will be rolled over into a new index.
                  global: True
                  advanced: True
-                  helpLink: elasticsearch
+                  helpLink: elasticsearch.html
              shrink:
                method:
                  description: Shrink the index to a new index with fewer primary shards. Shrink operation is by count or size.
@@ -482,7 +428,7 @@ elasticsearch:
              forcedType: string
              global: True
              advanced: True
-              helpLink: elasticsearch
+              helpLink: elasticsearch.html
            actions:
              set_priority:
                priority:
@@ -490,18 +436,18 @@ elasticsearch:
                  forcedType: int
                  global: True
                  advanced: True
-                  helpLink: elasticsearch
+                  helpLink: elasticsearch.html
              rollover:
                max_age:
                  description: Maximum age of index.  Once an index reaches this limit, it will be rolled over into a new index.
                  global: True
                  advanced: True
-                  helpLink: elasticsearch
+                  helpLink: elasticsearch.html
                max_primary_shard_size:
                  description: Maximum primary shard size. Once an index reaches this limit, it will be rolled over into a new index.
                  global: True
                  advanced: True
-                  helpLink: elasticsearch
+                  helpLink: elasticsearch.html
              shrink:
                method:
                  description: Shrink the index to a new index with fewer primary shards. Shrink operation is by count or size.
@@ -555,7 +501,7 @@ elasticsearch:
              forcedType: string
              global: True
              advanced: True
-              helpLink: elasticsearch
+              helpLink: elasticsearch.html
            actions:
              set_priority:
                priority:
@@ -563,7 +509,7 @@ elasticsearch:
                  forcedType: int
                  global: True
                  advanced: True
-                  helpLink: elasticsearch
+                  helpLink: elasticsearch.html
              allocate:
                number_of_replicas:
                  description: Set the number of replicas. Remains the same as the previous phase by default.
@@ -577,25 +523,25 @@ elasticsearch:
              forcedType: string
              global: True
              advanced: True
-              helpLink: elasticsearch
+              helpLink: elasticsearch.html
        _meta:
          package:
            name:
              description: Meta settings for the mapping.
              global: True
              advanced: True
-              helpLink: elasticsearch
+              helpLink: elasticsearch.html
          managed_by:
            description: Meta settings for the mapping.
            global: True
            advanced: True
-            helpLink: elasticsearch
+            helpLink: elasticsearch.html
          managed:
            description: Meta settings for the mapping.
            forcedType: bool
            global: True
            advanced: True
-            helpLink: elasticsearch
+            helpLink: elasticsearch.html
    so-logs-system_x_auth: *indexSettings
    so-logs-system_x_syslog: *indexSettings
    so-logs-system_x_system: *indexSettings
@@ -658,21 +604,20 @@ elasticsearch:
    so-metrics-fleet_server_x_agent_status: &fleetMetricsSettings
      index_sorting:
        description: Sorts the index by event time, at the cost of additional processing resource consumption.
-        forcedType: bool
        advanced: True
        readonly: True
-        helpLink: elasticsearch
+        helpLink: elasticsearch.html
      index_template:
        ignore_missing_component_templates:
          description: Ignore component templates if they aren't in Elasticsearch.
          advanced: True
          readonly: True
-          helpLink: elasticsearch
+          helpLink: elasticsearch.html
        index_patterns:
          description: Patterns for matching multiple indices or tables.
          advanced: True
          readonly: True
-          helpLink: elasticsearch
+          helpLink: elasticsearch.html
        template:
          settings:
            index:
@@ -680,35 +625,33 @@ elasticsearch:
                description: Type of mode used for this index. Time series indices can be used for metrics to reduce necessary storage.
                advanced: True
                readonly: True
-                helpLink: elasticsearch
+                helpLink: elasticsearch.html
              number_of_replicas:
                description: Number of replicas required for this index. Multiple replicas protects against data loss, but also increases storage costs.
                advanced: True
                readonly: True
-                helpLink: elasticsearch
+                helpLink: elasticsearch.html
        composed_of:
          description: The index template is composed of these component templates.
          advanced: True
          readonly: True
-          helpLink: elasticsearch
+          helpLink: elasticsearch.html
        priority:
          description: The priority of the index template.
          advanced: True
          readonly: True
-          helpLink: elasticsearch
+          helpLink: elasticsearch.html
        data_stream:
          hidden:
            description: Hide the data stream.
-            forcedType: bool
            advanced: True
            readonly: True
-            helpLink: elasticsearch
+            helpLink: elasticsearch.html
          allow_custom_routing:
            description: Allow custom routing for the data stream.
-            forcedType: bool
            advanced: True
            readonly: True
-            helpLink: elasticsearch
+            helpLink: elasticsearch.html
    so-metrics-fleet_server_x_agent_versions: *fleetMetricsSettings
  so_roles:
    so-manager: &soroleSettings
@@ -719,7 +662,7 @@ elasticsearch:
            forcedType: "[]string"
            global: False
            advanced: True
-            helpLink: elasticsearch
+            helpLink: elasticsearch.html
    so-managersearch: *soroleSettings
    so-standalone: *soroleSettings
    so-searchnode: *soroleSettings
@@ -14,42 +14,15 @@

 {% set ES_INDEX_SETTINGS_ORIG = ELASTICSEARCHDEFAULTS.elasticsearch.index_settings %}

-{% set ALL_ADDON_INTEGRATION_DEFAULTS = {} %}
-{% set ALL_ADDON_SETTINGS_ORIG = {} %}
-{% set ALL_ADDON_SETTINGS_GLOBAL_OVERRIDES = {} %}
-{% set ALL_ADDON_SETTINGS = {} %}
 {# start generation of integration default index_settings #}
-{% if salt['file.file_exists']('/opt/so/state/esfleet_component_templates.json') %}
-{#   import integration type defaults #}
-{%   if salt['file.file_exists']('/opt/so/state/esfleet_package_components.json') %}
-{%     set check_integration_package_components = salt['file.stats']('/opt/so/state/esfleet_package_components.json') %}
-{%     if check_integration_package_components.size > 1 %}
-{%       from 'elasticfleet/integration-defaults.map.jinja' import ADDON_INTEGRATION_DEFAULTS %}
-{%       do ALL_ADDON_INTEGRATION_DEFAULTS.update(ADDON_INTEGRATION_DEFAULTS) %}
-{%     endif %}
-{%   endif %}
-
-{#   import input type defaults #}
-{%   if salt['file.file_exists']('/opt/so/state/esfleet_input_package_components.json') %}
-{%     set check_input_package_components = salt['file.stats']('/opt/so/state/esfleet_input_package_components.json') %}
-{%     if check_input_package_components.size > 1 %}
-{%       from 'elasticfleet/input-defaults.map.jinja' import ADDON_INPUT_INTEGRATION_DEFAULTS %}
-{%       do ALL_ADDON_INTEGRATION_DEFAULTS.update(ADDON_INPUT_INTEGRATION_DEFAULTS) %}
-{%     endif %}
-{%   endif %}
-
-{#   import content type defaults #}
-{%   if salt['file.file_exists']('/opt/so/state/esfleet_content_package_components.json') %}
-{%     set check_content_package_components = salt['file.stats']('/opt/so/state/esfleet_content_package_components.json') %}
-{%     if check_content_package_components.size > 1 %}
-{%       from 'elasticfleet/content-defaults.map.jinja' import ADDON_CONTENT_INTEGRATION_DEFAULTS %}
-{%       do ALL_ADDON_INTEGRATION_DEFAULTS.update(ADDON_CONTENT_INTEGRATION_DEFAULTS) %}
-{%     endif %}
-{%   endif %}
-
-{%   for index, settings in ALL_ADDON_INTEGRATION_DEFAULTS.items() %}
-{%     do ALL_ADDON_SETTINGS_ORIG.update({index: settings}) %}
-{%   endfor %}
+{% if salt['file.file_exists']('/opt/so/state/esfleet_package_components.json') and salt['file.file_exists']('/opt/so/state/esfleet_component_templates.json') %}
+{%   set check_package_components = salt['file.stats']('/opt/so/state/esfleet_package_components.json') %}
+{%   if check_package_components.size > 1 %}
+{%    from 'elasticfleet/integration-defaults.map.jinja' import ADDON_INTEGRATION_DEFAULTS %}
+{%    for index, settings in ADDON_INTEGRATION_DEFAULTS.items() %}
+{%      do ES_INDEX_SETTINGS_ORIG.update({index: settings}) %}
+{%    endfor %}
+{%   endif%}
 {% endif %}
 {# end generation of integration default index_settings #}

@@ -58,33 +31,25 @@
 {%   do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.update({index: salt['defaults.merge'](ELASTICSEARCHDEFAULTS.elasticsearch.index_settings[index], PILLAR_GLOBAL_OVERRIDES, in_place=False)}) %}
 {% endfor %}

-{% if ALL_ADDON_SETTINGS_ORIG.keys() | length > 0 %}
-{%   for index in ALL_ADDON_SETTINGS_ORIG.keys() %}
-{%     do ALL_ADDON_SETTINGS_GLOBAL_OVERRIDES.update({index: salt['defaults.merge'](ALL_ADDON_SETTINGS_ORIG[index], PILLAR_GLOBAL_OVERRIDES, in_place=False)}) %}
-{%   endfor %}
-{% endif %}
-
 {% set ES_INDEX_SETTINGS = {} %}
-{% macro create_final_index_template(DEFINED_SETTINGS, GLOBAL_OVERRIDES, FINAL_INDEX_SETTINGS) %}
-
-{% do GLOBAL_OVERRIDES.update(salt['defaults.merge'](GLOBAL_OVERRIDES, ES_INDEX_PILLAR, in_place=False)) %}
-{% for index, settings in GLOBAL_OVERRIDES.items() %}
+{% do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.update(salt['defaults.merge'](ES_INDEX_SETTINGS_GLOBAL_OVERRIDES, ES_INDEX_PILLAR, in_place=False)) %}
+{% for index, settings in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.items() %}

 {#   prevent this action from being performed on custom defined indices. #}
 {#   the custom defined index is not present in either of the dictionaries and fails to reder. #}
-{%   if index in DEFINED_SETTINGS and index in GLOBAL_OVERRIDES %}
+{%   if index in ES_INDEX_SETTINGS_ORIG and index in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES %}

 {#     dont merge policy from the global_overrides if policy isn't defined in the original index settingss #}
 {#     this will prevent so-elasticsearch-ilm-policy-load from trying to load policy on non ILM manged indices #}
-{%     if not DEFINED_SETTINGS[index].policy is defined and GLOBAL_OVERRIDES[index].policy is defined %}
-{%       do GLOBAL_OVERRIDES[index].pop('policy') %}
+{%     if not ES_INDEX_SETTINGS_ORIG[index].policy is defined and ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy is defined %}
+{%       do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].pop('policy') %}
 {%     endif %}

 {#     this prevents and index from inderiting a policy phase from global overrides if it wasnt defined in the defaults. #}
-{%     if GLOBAL_OVERRIDES[index].policy is defined %}
-{%       for phase in GLOBAL_OVERRIDES[index].policy.phases.copy() %}
-{%         if DEFINED_SETTINGS[index].policy.phases[phase] is not defined %}
-{%           do GLOBAL_OVERRIDES[index].policy.phases.pop(phase) %}
+{%     if ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy is defined %}
+{%       for phase in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy.phases.copy() %}
+{%         if ES_INDEX_SETTINGS_ORIG[index].policy.phases[phase] is not defined %}
+{%           do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy.phases.pop(phase) %}
 {%         endif %}
 {%       endfor %}
 {%     endif %}
@@ -146,14 +111,5 @@
 {%     endfor %}
 {%   endif %}

-{%   do FINAL_INDEX_SETTINGS.update({index | replace("_x_", "."): GLOBAL_OVERRIDES[index]}) %}
+{%   do ES_INDEX_SETTINGS.update({index | replace("_x_", "."): ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index]}) %}
 {% endfor %}
-{% endmacro %}
-
-{{ create_final_index_template(ES_INDEX_SETTINGS_ORIG, ES_INDEX_SETTINGS_GLOBAL_OVERRIDES, ES_INDEX_SETTINGS) }}
-{{ create_final_index_template(ALL_ADDON_SETTINGS_ORIG, ALL_ADDON_SETTINGS_GLOBAL_OVERRIDES, ALL_ADDON_SETTINGS) }}
-
-{% set SO_MANAGED_INDICES = [] %}
-{% for index, settings in ES_INDEX_SETTINGS.items() %}
-{%   do SO_MANAGED_INDICES.append(index) %}
-{% endfor %}
@@ -6,19 +6,8 @@
 # Elastic License 2.0.

 . /usr/sbin/so-common
-
-if [[ -z "$1" ]]; then
-    if output=$(so-elasticsearch-query "_component_template" --retry 3 --retry-delay 1 --fail); then
-        jq '[.component_templates[] | .name] | sort' <<< "$output"
-    else
-        echo "Failed to retrieve component templates from Elasticsearch."
-        exit 1
-    fi
+if [ "$1" == "" ]; then
+    curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_component_template | jq '.component_templates[] |.name'| sort
 else
-    if output=$(so-elasticsearch-query "_component_template/$1" --retry 3 --retry-delay 1 --fail); then
-        jq <<< "$output"
-    else
-        echo "Failed to retrieve component template '$1' from Elasticsearch."
-        exit 1
-    fi
-fi
+    curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_component_template/$1 | jq
+fi
@@ -1,276 +0,0 @@
-#!/bin/bash
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-. /usr/sbin/so-common
-
-SO_STATEFILE_SUCCESS=/opt/so/state/estemplates.txt
-ADDON_STATEFILE_SUCCESS=/opt/so/state/addon_estemplates.txt
-ELASTICSEARCH_TEMPLATES_DIR="/opt/so/conf/elasticsearch/templates"
-SO_TEMPLATES_DIR="${ELASTICSEARCH_TEMPLATES_DIR}/index"
-ADDON_TEMPLATES_DIR="${ELASTICSEARCH_TEMPLATES_DIR}/addon-index"
-SO_LOAD_FAILURES=0
-ADDON_LOAD_FAILURES=0
-SO_LOAD_FAILURES_NAMES=()
-ADDON_LOAD_FAILURES_NAMES=()
-IS_HEAVYNODE="false"
-FORCE="false"
-VERBOSE="false"
-SHOULD_EXIT_ON_FAILURE="true"
-
-# If soup is running, ignore errors
-pgrep soup >/dev/null && SHOULD_EXIT_ON_FAILURE="false"
-
-while [[ $# -gt 0 ]]; do
-    case "$1" in
-    --heavynode)
-        IS_HEAVYNODE="true"
-        ;;
-    --force)
-        FORCE="true"
-        ;;
-    --verbose)
-        VERBOSE="true"
-        ;;
-    *)
-        echo "Usage: $0 [options]"
-        echo "Options:"
-        echo "  --heavynode     Only loads index templates specific to heavynodes"
-        echo "  --force     Force reload all templates regardless of statefiles (default: false)"
-        echo "  --verbose     Enable verbose output"
-        exit 1
-        ;;
-    esac
-    shift
-done
-
-load_template() {
-    local uri="$1"
-    local file="$2"
-
-    echo "Loading template file $file"
-    if ! output=$(retry 3 3 "so-elasticsearch-query $uri -d@$file -XPUT" "{\"acknowledged\":true}"); then
-        echo "$output"
-
-        return 1
-
-    elif [[ "$VERBOSE" == "true" ]]; then
-        echo "$output"
-    fi
-
-}
-
-check_required_component_template_exists() {
-    local required
-    local missing
-    local file=$1
-
-    required=$(jq '[((.composed_of //[]) - (.ignore_missing_component_templates // []))[]]' "$file")
-    missing=$(jq -n --argjson required "$required" --argjson component_templates "$component_templates" '(($required) - ($component_templates))')
-
-    if [[ $(jq length <<<"$missing") -gt 0 ]]; then
-
-        return 1
-    fi
-}
-
-check_heavynode_compatiable_index_template() {
-    # The only templates that are relevant to heavynodes are from datasets defined in elasticagent/files/elastic-agent.yml.jinja.
-    # Heavynodes do not have fleet server packages installed and do not support elastic agents reporting directly to them.
-    local -A heavynode_index_templates=(
-        ["so-import"]=1
-        ["so-syslog"]=1
-        ["so-logs-soc"]=1
-        ["so-suricata"]=1
-        ["so-suricata.alerts"]=1
-        ["so-zeek"]=1
-        ["so-strelka"]=1
-    )
-
-    local template_name="$1"
-
-    if [[ ! -v heavynode_index_templates["$template_name"] ]]; then
-
-        return 1
-    fi
-
-}
-
-load_component_templates() {
-    local printed_name="$1"
-    local pattern="${ELASTICSEARCH_TEMPLATES_DIR}/component/$2"
-    local append_mappings="${3:-"false"}"
-
-    echo -e "\nLoading $printed_name component templates...\n"
-
-    if ! compgen -G "${pattern}/*.json" > /dev/null; then
-        echo "No $printed_name component templates found in ${pattern}, skipping."
-        return
-    fi
-
-    for component in "$pattern"/*.json; do
-        tmpl_name=$(basename "${component%.json}")
-
-        if [[ "$append_mappings" == "true" ]]; then
-            # avoid duplicating "-mappings" if it already exists in the component template filename
-            tmpl_name="${tmpl_name%-mappings}-mappings"
-        fi
-
-        if ! load_template "_component_template/${tmpl_name}" "$component"; then
-            SO_LOAD_FAILURES=$((SO_LOAD_FAILURES + 1))
-            SO_LOAD_FAILURES_NAMES+=("$component")
-        fi
-    done
-}
-
-check_elasticsearch_responsive() {
-    # Cannot load templates if Elasticsearch is not responding.
-    #  NOTE: Slightly faster exit w/ failure than previous "retry 240 1" if there is a problem with Elasticsearch the
-    #    script should exit sooner rather than hang at the 'so-elasticsearch-templates' salt state.
-    retry 3 15 "so-elasticsearch-query / --output /dev/null --fail" ||
-        fail "Elasticsearch is not responding. Please review Elasticsearch logs /opt/so/log/elasticsearch/securityonion.log for more details. Additionally, consider running so-elasticsearch-troubleshoot."
-}
-
-index_templates_exist() {
-    local templates_dir="$1"
-
-    if [[ ! -d "$templates_dir" ]]; then
-        return 1
-    fi
-
-    compgen -G "${templates_dir}/*.json" > /dev/null
-}
-
-should_load_addon_templates() {
-    if [[ "$IS_HEAVYNODE" == "true" ]]; then
-        return 1
-    fi
-
-    # Skip statefile checks when forcing template load
-    if [[ "$FORCE" != "true" ]]; then
-        if [[ ! -f "$SO_STATEFILE_SUCCESS" || -f "$ADDON_STATEFILE_SUCCESS" ]]; then
-            return 1
-        fi
-    fi
-
-    index_templates_exist "$ADDON_TEMPLATES_DIR"
-}
-
-if [[ "$FORCE" == "true" || ! -f "$SO_STATEFILE_SUCCESS" ]] && index_templates_exist "$SO_TEMPLATES_DIR"; then
-    check_elasticsearch_responsive
-
-    if [[ "$IS_HEAVYNODE" == "false" ]]; then
-        # TODO: Better way to check if fleet server is installed vs checking for Elastic Defend component template.
-        fleet_check="logs-endpoint.alerts@package"
-        if ! so-elasticsearch-query "_component_template/$fleet_check" --output /dev/null --retry 5 --retry-delay 3 --fail; then
-            # This check prevents so-elasticsearch-templates-load from running before so-elastic-fleet-setup has run.
-            echo -e "\nPackage $fleet_check not yet installed. Fleet Server may not be fully configured yet."
-            # Fleet Server is required because some SO index templates depend on components installed via
-            #  specific integrations eg Elastic Defend. These are components that we do not manually create / manage
-            #  via /opt/so/saltstack/salt/elasticsearch/templates/component/
-
-            exit 0
-        fi
-    fi
-
-    # load_component_templates "Name" "directory" "append '-mappings'?"
-    load_component_templates "ECS" "ecs" "true"
-    load_component_templates "Elastic Agent" "elastic-agent"
-    load_component_templates "Security Onion" "so"
-
-    component_templates=$(so-elasticsearch-component-templates-list)
-    echo -e "Loading Security Onion index templates...\n"
-    for so_idx_tmpl in "${SO_TEMPLATES_DIR}"/*.json; do
-        tmpl_name=$(basename "${so_idx_tmpl%-template.json}")
-
-        if [[ "$IS_HEAVYNODE" == "true" ]]; then
-            # TODO: Better way to load only heavynode specific templates
-            if ! check_heavynode_compatiable_index_template "$tmpl_name"; then
-                if [[ "$VERBOSE" == "true" ]]; then
-                    echo "Skipping over $so_idx_tmpl, template is not a heavynode specific index template."
-                fi
-
-                continue
-            fi
-        fi
-
-        if check_required_component_template_exists "$so_idx_tmpl"; then
-            if ! load_template "_index_template/$tmpl_name" "$so_idx_tmpl"; then
-                SO_LOAD_FAILURES=$((SO_LOAD_FAILURES + 1))
-                SO_LOAD_FAILURES_NAMES+=("$so_idx_tmpl")
-            fi
-        else
-            echo "Skipping over $so_idx_tmpl due to missing required component template(s)."
-            SO_LOAD_FAILURES=$((SO_LOAD_FAILURES + 1))
-            SO_LOAD_FAILURES_NAMES+=("$so_idx_tmpl")
-
-            continue
-        fi
-    done
-
-    if [[ $SO_LOAD_FAILURES -eq 0 ]]; then
-        echo "All Security Onion core templates loaded successfully."
-
-        touch "$SO_STATEFILE_SUCCESS"
-    else
-        echo "Encountered $SO_LOAD_FAILURES failure(s) loading templates:"
-        for failed_template in "${SO_LOAD_FAILURES_NAMES[@]}"; do
-            echo "  - $failed_template"
-        done
-        if [[ "$SHOULD_EXIT_ON_FAILURE" == "true" ]]; then
-            fail "Failed to load all Security Onion core templates successfully."
-        fi
-    fi
-elif ! index_templates_exist "$SO_TEMPLATES_DIR"; then
-    echo "No Security Onion core index templates found in ${SO_TEMPLATES_DIR}, skipping."
-elif [[ -f "$SO_STATEFILE_SUCCESS" ]]; then
-    echo "Security Onion core templates already loaded"
-fi
-
-# Start loading addon templates
-if should_load_addon_templates; then
-
-    check_elasticsearch_responsive
-
-    echo -e "\nLoading addon integration index templates...\n"
-    component_templates=$(so-elasticsearch-component-templates-list)
-
-    for addon_idx_tmpl in "${ADDON_TEMPLATES_DIR}"/*.json; do
-        tmpl_name=$(basename "${addon_idx_tmpl%-template.json}")
-
-        if check_required_component_template_exists "$addon_idx_tmpl"; then
-            if ! load_template "_index_template/${tmpl_name}" "$addon_idx_tmpl"; then
-                ADDON_LOAD_FAILURES=$((ADDON_LOAD_FAILURES + 1))
-                ADDON_LOAD_FAILURES_NAMES+=("$addon_idx_tmpl")
-            fi
-        else
-            echo "Skipping over $addon_idx_tmpl due to missing required component template(s)."
-            ADDON_LOAD_FAILURES=$((ADDON_LOAD_FAILURES + 1))
-            ADDON_LOAD_FAILURES_NAMES+=("$addon_idx_tmpl")
-
-            continue
-        fi
-    done
-
-    if [[ $ADDON_LOAD_FAILURES -eq 0 ]]; then
-        echo "All addon integration templates loaded successfully."
-
-        touch "$ADDON_STATEFILE_SUCCESS"
-    else
-        echo "Encountered $ADDON_LOAD_FAILURES failure(s) loading addon integration templates:"
-        for failed_template in "${ADDON_LOAD_FAILURES_NAMES[@]}"; do
-            echo "  - $failed_template"
-        done
-        if [[ "$SHOULD_EXIT_ON_FAILURE" == "true" ]]; then
-            fail "Failed to load all addon integration templates successfully."
-        fi
-    fi
-
-elif [[ ! -f "$SO_STATEFILE_SUCCESS" && "$IS_HEAVYNODE" == "false" ]]; then
-    echo "Skipping loading addon integration templates until Security Onion core templates have been loaded."
-
-elif [[ -f "$ADDON_STATEFILE_SUCCESS" && "$IS_HEAVYNODE" == "false" && "$FORCE" == "false" ]]; then
-    echo "Addon integration templates already loaded"
-fi
@@ -7,9 +7,6 @@
 . /usr/sbin/so-common

 {%- from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS %}
-{%- if GLOBALS.role != "so-heavynode" %}
-{%-   from 'elasticsearch/template.map.jinja' import ALL_ADDON_SETTINGS %}
-{%- endif %}

 {%- for index, settings in ES_INDEX_SETTINGS.items() %}
 {%-   if settings.policy is defined %}
@@ -36,13 +33,3 @@
 {%-   endif %}
 {%- endfor %}
 echo
-{%- if GLOBALS.role != "so-heavynode" %}
-{%-   for index, settings in ALL_ADDON_SETTINGS.items() %}
-{%-     if settings.policy is defined %}
-  echo
-  echo "Setting up {{ index }}-logs policy..."
-  curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://localhost:9200/_ilm/policy/{{ index }}-logs" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }'
-  echo
-{%-     endif %}
-{%-   endfor %}
-{%- endif %}
@@ -0,0 +1,165 @@
+#!/bin/bash
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+{%- import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %}
+{%  from 'vars/globals.map.jinja' import GLOBALS %}
+
+STATE_FILE_INITIAL=/opt/so/state/estemplates_initial_load_attempt.txt
+STATE_FILE_SUCCESS=/opt/so/state/estemplates.txt
+
+if [[ -f $STATE_FILE_INITIAL ]]; then
+  # The initial template load has already run. As this is a subsequent load, all dependencies should 
+  # already be satisified. Therefore, immediately exit/abort this script upon any template load failure
+  # since this is an unrecoverable failure.
+  should_exit_on_failure=1
+else
+  # This is the initial template load, and there likely are some components not yet setup in Elasticsearch.
+  # Therefore load as many templates as possible at this time and if an error occurs proceed to the next 
+  # template. But if at least one template fails to load do not mark the templates as having been loaded.
+  # This will allow the next load to resume the load of the templates that failed to load initially.
+  should_exit_on_failure=0
+  echo "This is the initial template load"
+fi
+
+# If soup is running, ignore errors
+pgrep soup > /dev/null && should_exit_on_failure=0
+
+load_failures=0
+
+load_template() {
+  uri=$1
+  file=$2
+
+  echo "Loading template file $i" 
+  if ! retry 3 1 "so-elasticsearch-query $uri -d@$file -XPUT" "{\"acknowledged\":true}"; then
+    if [[ $should_exit_on_failure -eq 1 ]]; then
+      fail "Could not load template file: $file"
+    else
+      load_failures=$((load_failures+1))
+      echo "Incremented load failure counter: $load_failures"
+    fi
+  fi
+}
+
+if [ ! -f $STATE_FILE_SUCCESS ]; then
+  echo "State file $STATE_FILE_SUCCESS not found. Running so-elasticsearch-templates-load."
+
+  . /usr/sbin/so-common
+
+  {% if GLOBALS.role != 'so-heavynode' %}
+  if [ -f /usr/sbin/so-elastic-fleet-common ]; then
+    . /usr/sbin/so-elastic-fleet-common
+  fi
+  {% endif %}
+
+  default_conf_dir=/opt/so/conf
+
+  # Define a default directory to load pipelines from
+  ELASTICSEARCH_TEMPLATES="$default_conf_dir/elasticsearch/templates/"
+
+  {% if GLOBALS.role == 'so-heavynode' %}
+  file="/opt/so/conf/elasticsearch/templates/index/so-common-template.json"
+  {% else %}
+  file="/usr/sbin/so-elastic-fleet-common"
+  {% endif %}
+
+  if [ -f "$file" ]; then
+    # Wait for ElasticSearch to initialize
+    echo -n "Waiting for ElasticSearch..."
+    retry 240 1 "so-elasticsearch-query / -k --output /dev/null --silent --head --fail" || fail "Connection attempt timed out.  Unable to connect to ElasticSearch.  \nPlease try: \n  -checking log(s) in /var/log/elasticsearch/\n  -running 'sudo docker ps' \n  -running 'sudo so-elastic-restart'"
+    {% if GLOBALS.role != 'so-heavynode' %}
+    TEMPLATE="logs-endpoint.alerts@package"
+    INSTALLED=$(so-elasticsearch-query _component_template/$TEMPLATE | jq -r .component_templates[0].name)
+    if [ "$INSTALLED" != "$TEMPLATE" ]; then
+      echo
+      echo "Packages not yet installed."
+      echo
+      exit 0
+    fi
+    {% endif %}
+
+    touch $STATE_FILE_INITIAL
+
+    cd ${ELASTICSEARCH_TEMPLATES}/component/ecs
+
+    echo "Loading ECS component templates..."
+    for i in *; do 
+      TEMPLATE=$(echo $i | cut -d '.' -f1) 
+      load_template "_component_template/${TEMPLATE}-mappings" "$i"
+    done
+    echo
+
+    cd ${ELASTICSEARCH_TEMPLATES}/component/elastic-agent
+
+    echo "Loading Elastic Agent component templates..."
+    {% if GLOBALS.role == 'so-heavynode' %}
+    component_pattern="so-*"
+    {% else %}
+    component_pattern="*"
+    {% endif %}
+    for i in $component_pattern; do 
+      TEMPLATE=${i::-5} 
+      load_template "_component_template/$TEMPLATE" "$i"
+    done
+    echo
+	    
+    # Load SO-specific component templates
+    cd ${ELASTICSEARCH_TEMPLATES}/component/so
+
+    echo "Loading Security Onion component templates..."
+    for i in *; do 
+      TEMPLATE=$(echo $i | cut -d '.' -f1); 
+      load_template "_component_template/$TEMPLATE" "$i"
+    done
+    echo
+
+    # Load SO index templates
+    cd ${ELASTICSEARCH_TEMPLATES}/index
+
+    echo "Loading Security Onion index templates..."
+    shopt -s extglob
+    {% if GLOBALS.role == 'so-heavynode' %}
+    pattern="!(*1password*|*aws*|*azure*|*cloudflare*|*elastic_agent*|*fim*|*github*|*google*|*osquery*|*system*|*windows*|*endpoint*|*elasticsearch*|*generic*|*fleet_server*|*soc*)"
+    {% else %}
+    pattern="*"
+    {% endif %}
+    # Index templates will be skipped if the following conditions are met:
+    # 1. The template is part of the "so-logs-" template group
+    # 2. The template name does not correlate to at least one existing component template
+    # In this situation, the script will treat the skipped template as a temporary failure
+    # and allow the templates to be loaded again on the next run or highstate, whichever 
+    # comes first.
+    COMPONENT_LIST=$(so-elasticsearch-component-templates-list)
+    for i in $pattern; do
+      TEMPLATE=${i::-14}
+      COMPONENT_PATTERN=${TEMPLATE:3}
+      MATCH=$(echo "$TEMPLATE" | grep -E "^so-logs-|^so-metrics" | grep -vE "detections|osquery")
+      if [[ -n "$MATCH" && ! "$COMPONENT_LIST" =~ "$COMPONENT_PATTERN" && ! "$COMPONENT_PATTERN" =~ \.generic|logs-winlog\.winlog ]]; then
+        load_failures=$((load_failures+1))
+        echo "Component template does not exist for $COMPONENT_PATTERN. The index template will not be loaded. Load failures: $load_failures"
+      else
+        load_template "_index_template/$TEMPLATE" "$i"
+      fi
+    done
+  else
+    {% if GLOBALS.role == 'so-heavynode' %}
+    echo "Common template does not exist. Exiting..."
+    {% else %}
+    echo "Elastic Fleet not configured. Exiting..."
+    {% endif %}
+    exit 0
+  fi
+  
+  cd - >/dev/null
+
+  if [[ $load_failures -eq 0 ]]; then
+    echo "All templates loaded successfully"
+    touch $STATE_FILE_SUCCESS
+  else
+    echo "Encountered $load_failures templates that were unable to load, likely due to missing dependencies that will be available later; will retry on next highstate"
+  fi
+else
+  echo "Templates already loaded"
+fi
@@ -11,7 +11,6 @@
     'so-kratos',
     'so-hydra',
     'so-nginx',
-     'so-postgres',
     'so-redis',
     'so-soc',
     'so-strelka-coordinator',
@@ -35,7 +34,6 @@
     'so-hydra',
     'so-logstash',
     'so-nginx',
-     'so-postgres',
     'so-redis',
     'so-soc',
     'so-strelka-coordinator',
@@ -79,7 +77,6 @@
     'so-kratos',
     'so-hydra',
     'so-nginx',
-     'so-postgres',
     'so-soc'
 ] %}

@@ -98,10 +98,6 @@ firewall:
      tcp:
        - 8086
      udp: []
-    postgres:
-      tcp:
-        - 5432
-      udp: []
    kafka_controller:
      tcp:
        - 9093
@@ -197,7 +193,6 @@ firewall:
                - kibana
                - redis
                - influxdb
-                - postgres
                - elasticsearch_rest
                - elasticsearch_node
                - localrules
@@ -384,7 +379,6 @@ firewall:
                - kibana
                - redis
                - influxdb
-                - postgres
                - elasticsearch_rest
                - elasticsearch_node
                - docker_registry
@@ -398,7 +392,6 @@ firewall:
                - elasticsearch_rest
                - docker_registry
                - influxdb
-                - postgres
                - sensoroni
                - yum
                - beats_5044
@@ -411,7 +404,6 @@ firewall:
              portgroups:
                - docker_registry
                - influxdb
-                - postgres
                - sensoroni
                - yum
                - beats_5044
@@ -429,7 +421,6 @@ firewall:
                - yum
                - docker_registry
                - influxdb
-                - postgres
                - sensoroni
            searchnode:
              portgroups:
@@ -440,7 +431,6 @@ firewall:
                - yum
                - docker_registry
                - influxdb
-                - postgres
                - elastic_agent_control
                - elastic_agent_data
                - elastic_agent_update
@@ -454,7 +444,6 @@ firewall:
                - yum
                - docker_registry
                - influxdb
-                - postgres
                - elastic_agent_control
                - elastic_agent_data
                - elastic_agent_update
@@ -464,7 +453,6 @@ firewall:
                - yum
                - docker_registry
                - influxdb
-                - postgres
                - elastic_agent_control
                - elastic_agent_data
                - elastic_agent_update
@@ -498,7 +486,6 @@ firewall:
              portgroups:
                - docker_registry
                - influxdb
-                - postgres
                - sensoroni
                - yum
                - elastic_agent_control
@@ -509,7 +496,6 @@ firewall:
                - yum
                - docker_registry
                - influxdb
-                - postgres
                - elastic_agent_control
                - elastic_agent_data
                - elastic_agent_update
@@ -604,7 +590,6 @@ firewall:
                - kibana
                - redis
                - influxdb
-                - postgres
                - elasticsearch_rest
                - elasticsearch_node
                - docker_registry
@@ -618,7 +603,6 @@ firewall:
                - elasticsearch_rest
                - docker_registry
                - influxdb
-                - postgres
                - sensoroni
                - yum
                - beats_5044
@@ -631,7 +615,6 @@ firewall:
              portgroups:
                - docker_registry
                - influxdb
-                - postgres
                - sensoroni
                - yum
                - beats_5044
@@ -649,7 +632,6 @@ firewall:
                - yum
                - docker_registry
                - influxdb
-                - postgres
                - sensoroni
            searchnode:
              portgroups:
@@ -660,7 +642,6 @@ firewall:
                - yum
                - docker_registry
                - influxdb
-                - postgres
                - elastic_agent_control
                - elastic_agent_data
                - elastic_agent_update
@@ -674,7 +655,6 @@ firewall:
                - yum
                - docker_registry
                - influxdb
-                - postgres
                - elastic_agent_control
                - elastic_agent_data
                - elastic_agent_update
@@ -684,7 +664,6 @@ firewall:
                - yum
                - docker_registry
                - influxdb
-                - postgres
                - elastic_agent_control
                - elastic_agent_data
                - elastic_agent_update
@@ -716,7 +695,6 @@ firewall:
              portgroups:
                - docker_registry
                - influxdb
-                - postgres
                - sensoroni
                - yum
                - elastic_agent_control
@@ -727,7 +705,6 @@ firewall:
                - yum
                - docker_registry
                - influxdb
-                - postgres
                - elastic_agent_control
                - elastic_agent_data
                - elastic_agent_update
@@ -822,7 +799,6 @@ firewall:
                - kibana
                - redis
                - influxdb
-                - postgres
                - elasticsearch_rest
                - elasticsearch_node
                - docker_registry
@@ -836,7 +812,6 @@ firewall:
                - elasticsearch_rest
                - docker_registry
                - influxdb
-                - postgres
                - sensoroni
                - yum
                - beats_5044
@@ -849,7 +824,6 @@ firewall:
              portgroups:
                - docker_registry
                - influxdb
-                - postgres
                - sensoroni
                - yum
                - beats_5044
@@ -867,7 +841,6 @@ firewall:
                - yum
                - docker_registry
                - influxdb
-                - postgres
                - sensoroni
            searchnode:
              portgroups:
@@ -877,7 +850,6 @@ firewall:
                - yum
                - docker_registry
                - influxdb
-                - postgres
                - elastic_agent_control
                - elastic_agent_data
                - elastic_agent_update
@@ -890,7 +862,6 @@ firewall:
                - yum
                - docker_registry
                - influxdb
-                - postgres
                - elastic_agent_control
                - elastic_agent_data
                - elastic_agent_update
@@ -900,7 +871,6 @@ firewall:
                - yum
                - docker_registry
                - influxdb
-                - postgres
                - elastic_agent_control
                - elastic_agent_data
                - elastic_agent_update
@@ -934,7 +904,6 @@ firewall:
              portgroups:
                - docker_registry
                - influxdb
-                - postgres
                - sensoroni
                - yum
                - elastic_agent_control
@@ -945,7 +914,6 @@ firewall:
                - yum
                - docker_registry
                - influxdb
-                - postgres
                - elastic_agent_control
                - elastic_agent_data
                - elastic_agent_update
@@ -1043,7 +1011,6 @@ firewall:
                - kibana
                - redis
                - influxdb
-                - postgres
                - elasticsearch_rest
                - elasticsearch_node
                - docker_registry
@@ -1064,7 +1031,6 @@ firewall:
                - elasticsearch_rest
                - docker_registry
                - influxdb
-                - postgres
                - sensoroni
                - yum
                - beats_5044
@@ -1077,7 +1043,6 @@ firewall:
              portgroups:
                - docker_registry
                - influxdb
-                - postgres
                - sensoroni
                - yum
                - beats_5044
@@ -1089,7 +1054,6 @@ firewall:
              portgroups:
                - docker_registry
                - influxdb
-                - postgres
                - sensoroni
                - yum
                - beats_5044
@@ -1101,7 +1065,6 @@ firewall:
              portgroups:
                - docker_registry
                - influxdb
-                - postgres
                - sensoroni
                - yum
                - redis
@@ -1111,7 +1074,6 @@ firewall:
              portgroups:
                - docker_registry
                - influxdb
-                - postgres
                - sensoroni
                - yum
                - redis
@@ -1122,7 +1084,6 @@ firewall:
                - yum
                - docker_registry
                - influxdb
-                - postgres
                - elastic_agent_control
                - elastic_agent_data
                - elastic_agent_update
@@ -1159,7 +1120,6 @@ firewall:
              portgroups:
                - docker_registry
                - influxdb
-                - postgres
                - sensoroni
                - yum
                - elastic_agent_control
@@ -1170,7 +1130,6 @@ firewall:
                - yum
                - docker_registry
                - influxdb
-                - postgres
                - elastic_agent_control
                - elastic_agent_data
                - elastic_agent_update
@@ -1514,7 +1473,6 @@ firewall:
                - kibana
                - redis
                - influxdb
-                - postgres
                - elasticsearch_rest
                - elasticsearch_node
                - elastic_agent_control
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Mike Reeves	55af7eb541	Merge pull request #15732 from Security-Onion-Solutions/hotfix/2.4.211 Hotfix 2.4.211	2026-04-07 13:19:57 -04:00
Mike Reeves	ece7cdac04	Merge pull request #15731 from Security-Onion-Solutions/hf211 2.4.211 hotfix	2026-04-07 11:38:52 -04:00
Mike Reeves	b63d5e4c83	2.4.211 hotfix	2026-04-07 11:27:57 -04:00
Mike Reeves	0f97249009	Merge pull request #15726 from Security-Onion-Solutions/cherry-pick/suricata-bpf-hotfix-2.4.211 Cherry-pick suricata bpf fix and HOTFIX update	2026-04-06 14:29:00 -04:00
Josh Patterson	043feee4c0	update HOTFIX	2026-04-06 14:26:25 -04:00
Josh Patterson	c3707dc814	fix suricata bpf for transition mode	2026-04-06 14:26:25 -04:00
Mike Reeves	4afd186ec7	Merge pull request #15714 from Security-Onion-Solutions/soupfix2 Fix soup for 3.X.X installations	2026-04-01 17:20:37 -04:00
Mike Reeves	3f27d00dd1	Fix soup_scripts.sls to handle 3.X versions instead of falling back to 2.3	2026-04-01 16:12:07 -04:00
Mike Reeves	f2d6d8a3fe	Fetch and replace soup scripts from 3/main before proceeding on 3.X.X	2026-04-01 15:43:21 -04:00
Mike Reeves	1355e8eb11	Re-exec soup with 3/main branch when running on 3.X.X	2026-04-01 15:27:24 -04:00
Mike Reeves	472769452c	Default soup to 3/main branch for 3.X.X installations	2026-04-01 15:22:37 -04:00
Mike Reeves	89e470059e	Merge pull request #15597 from Security-Onion-Solutions/2.4.211 2.4.211	2026-03-12 13:18:19 -04:00
Mike Reeves	79b30e43d9	2.4.211	2026-03-12 11:33:11 -04:00
Mike Reeves	5cebce32f7	2.4.211	2026-03-12 11:31:59 -04:00
Josh Patterson	810681c92e	Merge pull request #15593 from Security-Onion-Solutions/ulimit set container ulimits to default	2026-03-11 14:40:40 -04:00
Josh Patterson	51f9104d0f	set container ulimits to default	2026-03-11 14:37:43 -04:00
Mike Reeves	8da5ed673b	Merge pull request #15586 from Security-Onion-Solutions/TOoSmOotH-patch-4 Add support for version 2.4.211 in soup script	2026-03-11 12:16:49 -04:00
Josh Patterson	83ba40b548	Merge pull request #15588 from Security-Onion-Solutions/m0duspwnens-patch-1 clear HOTFIX file	2026-03-11 12:16:21 -04:00
Josh Patterson	7de8528b34	clear HOTFIX file	2026-03-11 12:14:48 -04:00
Mike Reeves	e6bd57e08d	Fix conditional check for POSTVERSION 2.4.211	2026-03-11 12:13:05 -04:00
Mike Reeves	06664440ad	Add support for version 2.4.211 in soup script	2026-03-11 12:10:28 -04:00
Josh Patterson	bd31f2898b	Merge pull request #15584 from Security-Onion-Solutions/hypefix remove 10T virtual disk limit. URL_BASE to vm hosts file	2026-03-11 11:58:46 -04:00
Josh Patterson	5bf9d92b52	add URL_BASE to vm hosts file	2026-03-11 11:55:42 -04:00
Josh Patterson	48c369ed11	remove 10T limit for virtual disk	2026-03-11 11:55:01 -04:00
Josh Patterson	7fec2d59a7	Merge pull request #15583 from Security-Onion-Solutions/m0duspwnens-patch-1 fix enable/disable suricata pcap	2026-03-11 11:52:53 -04:00
Mike Reeves	a0ad589c3a	Merge pull request #15582 from Security-Onion-Solutions/TOoSmOotH-patch-3 Bump version from 2.4.210 to 2.4.211	2026-03-11 11:48:51 -04:00
Mike Reeves	0bd54e2835	Add version 2.4.211 to discussion template	2026-03-11 11:44:57 -04:00
Mike Reeves	58f5c56b72	Bump version from 2.4.210 to 2.4.211	2026-03-11 11:43:42 -04:00
Josh Patterson	6472c610d0	fix enable/disable suricata pcap suricata pcap can now be enabled/disabled through pcap:enabled grid config / pillar	2026-03-10 11:01:11 -04:00
Mike Reeves	179c1ea7f7	Merge pull request #15570 from Security-Onion-Solutions/TOoSmOotH-patch-1 Add date to HOTFIX file	2026-03-10 10:20:16 -04:00
Mike Reeves	db964cad21	Add date to HOTFIX file	2026-03-10 10:18:25 -04:00
@@ -1 +1 @@
 .1.0
 .4.211