remove foxtrot version

Merge pull request #15775 from Security-Onion-Solutions/reyesj2-es932
check for addon-index templates dir before attempting to load addon i…
2026-04-24 21:47:48 +02:00 · 2026-04-15 15:04:20 -05:00 · 2026-04-14 19:27:02 -05:00 · 2026-04-14 19:26:37 -05:00 · 2026-04-14 15:04:46 -05:00 · 2026-04-14 14:55:33 -05:00
563 changed files with 88409 additions and 51751 deletions
@@ -1,546 +0,0 @@
 title = "gitleaks config"
 # Gitleaks rules are defined by regular expressions and entropy ranges.
 # Some secrets have unique signatures which make detecting those secrets easy.
 # Examples of those secrets would be GitLab Personal Access Tokens, AWS keys, and GitHub Access Tokens.
 # All these examples have defined prefixes like `glpat`, `AKIA`, `ghp_`, etc.
 #
 # Other secrets might just be a hash which means we need to write more complex rules to verify
 # that what we are matching is a secret.
 #
 # Here is an example of a semi-generic secret
 #
 #   discord_client_secret = "8dyfuiRyq=vVc3RRr_edRk-fK__JItpZ"
 #
 # We can write a regular expression to capture the variable name (identifier),
 # the assignment symbol (like '=' or ':='), and finally the actual secret.
 # The structure of a rule to match this example secret is below:
 #
 #                                                           Beginning string
 #                                                               quotation
 #                                                                   │            End string quotation
 #                                                                   │                      │
 #                                                                   ▼                      ▼
 #    (?i)(discord[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9=_\-]{32})['\"]
 #
 #                   ▲                              ▲                                ▲
 #                   │                              │                                │
 #                   │                              │                                │
 #              identifier                  assignment symbol
 #                                                                                Secret
 #
 [[rules]]
 id = "gitlab-pat"
 description = "GitLab Personal Access Token"
 regex = '''glpat-[0-9a-zA-Z\-\_]{20}'''
 [[rules]]
 id = "aws-access-token"
 description = "AWS"
 regex = '''(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}'''
 # Cryptographic keys
 [[rules]]
 id = "PKCS8-PK"
 description = "PKCS8 private key"
 regex = '''-----BEGIN PRIVATE KEY-----'''
 [[rules]]
 id = "RSA-PK"
 description = "RSA private key"
 regex = '''-----BEGIN RSA PRIVATE KEY-----'''
 [[rules]]
 id = "OPENSSH-PK"
 description = "SSH private key"
 regex = '''-----BEGIN OPENSSH PRIVATE KEY-----'''
 [[rules]]
 id = "PGP-PK"
 description = "PGP private key"
 regex = '''-----BEGIN PGP PRIVATE KEY BLOCK-----'''
 [[rules]]
 id = "github-pat"
 description = "GitHub Personal Access Token"
 regex = '''ghp_[0-9a-zA-Z]{36}'''
 [[rules]]
 id = "github-oauth"
 description = "GitHub OAuth Access Token"
 regex = '''gho_[0-9a-zA-Z]{36}'''
 [[rules]]
 id = "SSH-DSA-PK"
 description = "SSH (DSA) private key"
 regex = '''-----BEGIN DSA PRIVATE KEY-----'''
 [[rules]]
 id = "SSH-EC-PK"
 description = "SSH (EC) private key"
 regex = '''-----BEGIN EC PRIVATE KEY-----'''
 [[rules]]
 id = "github-app-token"
 description = "GitHub App Token"
 regex = '''(ghu|ghs)_[0-9a-zA-Z]{36}'''
 [[rules]]
 id = "github-refresh-token"
 description = "GitHub Refresh Token"
 regex = '''ghr_[0-9a-zA-Z]{76}'''
 [[rules]]
 id = "shopify-shared-secret"
 description = "Shopify shared secret"
 regex = '''shpss_[a-fA-F0-9]{32}'''
 [[rules]]
 id = "shopify-access-token"
 description = "Shopify access token"
 regex = '''shpat_[a-fA-F0-9]{32}'''
 [[rules]]
 id = "shopify-custom-access-token"
 description = "Shopify custom app access token"
 regex = '''shpca_[a-fA-F0-9]{32}'''
 [[rules]]
 id = "shopify-private-app-access-token"
 description = "Shopify private app access token"
 regex = '''shppa_[a-fA-F0-9]{32}'''
 [[rules]]
 id = "slack-access-token"
 description = "Slack token"
 regex = '''xox[baprs]-([0-9a-zA-Z]{10,48})?'''
 [[rules]]
 id = "stripe-access-token"
 description = "Stripe"
 regex = '''(?i)(sk|pk)_(test|live)_[0-9a-z]{10,32}'''
 [[rules]]
 id = "pypi-upload-token"
 description = "PyPI upload token"
 regex = '''pypi-AgEIcHlwaS5vcmc[A-Za-z0-9\-_]{50,1000}'''
 [[rules]]
 id = "gcp-service-account"
 description = "Google (GCP) Service-account"
 regex = '''\"type\": \"service_account\"'''
 [[rules]]
 id = "heroku-api-key"
 description = "Heroku API Key"
 regex = ''' (?i)(heroku[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([0-9A-F]{8}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{12})['\"]'''
 secretGroup = 3
 [[rules]]
 id = "slack-web-hook"
 description = "Slack Webhook"
 regex = '''https://hooks.slack.com/services/T[a-zA-Z0-9_]{8}/B[a-zA-Z0-9_]{8,12}/[a-zA-Z0-9_]{24}'''
 [[rules]]
 id = "twilio-api-key"
 description = "Twilio API Key"
 regex = '''SK[0-9a-fA-F]{32}'''
 [[rules]]
 id = "age-secret-key"
 description = "Age secret key"
 regex = '''AGE-SECRET-KEY-1[QPZRY9X8GF2TVDW0S3JN54KHCE6MUA7L]{58}'''
 [[rules]]
 id = "facebook-token"
 description = "Facebook token"
 regex = '''(?i)(facebook[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-f0-9]{32})['\"]'''
 secretGroup = 3
 [[rules]]
 id = "twitter-token"
 description = "Twitter token"
 regex = '''(?i)(twitter[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-f0-9]{35,44})['\"]'''
 secretGroup = 3
 [[rules]]
 id = "adobe-client-id"
 description = "Adobe Client ID (Oauth Web)"
 regex = '''(?i)(adobe[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-f0-9]{32})['\"]'''
 secretGroup = 3
 [[rules]]
 id = "adobe-client-secret"
 description = "Adobe Client Secret"
 regex = '''(p8e-)(?i)[a-z0-9]{32}'''
 [[rules]]
 id = "alibaba-access-key-id"
 description = "Alibaba AccessKey ID"
 regex = '''(LTAI)(?i)[a-z0-9]{20}'''
 [[rules]]
 id = "alibaba-secret-key"
 description = "Alibaba Secret Key"
 regex = '''(?i)(alibaba[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{30})['\"]'''
 secretGroup = 3
 [[rules]]
 id = "asana-client-id"
 description = "Asana Client ID"
 regex = '''(?i)(asana[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([0-9]{16})['\"]'''
 secretGroup = 3
 [[rules]]
 id = "asana-client-secret"
 description = "Asana Client Secret"
 regex = '''(?i)(asana[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{32})['\"]'''
 secretGroup = 3
 [[rules]]
 id = "atlassian-api-token"
 description = "Atlassian API token"
 regex = '''(?i)(atlassian[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{24})['\"]'''
 secretGroup = 3
 [[rules]]
 id = "bitbucket-client-id"
 description = "Bitbucket client ID"
 regex = '''(?i)(bitbucket[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{32})['\"]'''
 secretGroup = 3
 [[rules]]
 id = "bitbucket-client-secret"
 description = "Bitbucket client secret"
 regex = '''(?i)(bitbucket[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9_\-]{64})['\"]'''
 secretGroup = 3
 [[rules]]
 id = "beamer-api-token"
 description = "Beamer API token"
 regex = '''(?i)(beamer[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](b_[a-z0-9=_\-]{44})['\"]'''
 secretGroup = 3
 [[rules]]
 id = "clojars-api-token"
 description = "Clojars API token"
 regex = '''(CLOJARS_)(?i)[a-z0-9]{60}'''
 [[rules]]
 id = "contentful-delivery-api-token"
 description = "Contentful delivery API token"
 regex = '''(?i)(contentful[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9\-=_]{43})['\"]'''
 secretGroup = 3
 [[rules]]
 id = "databricks-api-token"
 description = "Databricks API token"
 regex = '''dapi[a-h0-9]{32}'''
 [[rules]]
 id = "discord-api-token"
 description = "Discord API key"
 regex = '''(?i)(discord[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-h0-9]{64})['\"]'''
 secretGroup = 3
 [[rules]]
 id = "discord-client-id"
 description = "Discord client ID"
 regex = '''(?i)(discord[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([0-9]{18})['\"]'''
 secretGroup = 3
 [[rules]]
 id = "discord-client-secret"
 description = "Discord client secret"
 regex = '''(?i)(discord[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9=_\-]{32})['\"]'''
 secretGroup = 3
 [[rules]]
 id = "doppler-api-token"
 description = "Doppler API token"
 regex = '''['\"](dp\.pt\.)(?i)[a-z0-9]{43}['\"]'''
 [[rules]]
 id = "dropbox-api-secret"
 description = "Dropbox API secret/key"
 regex = '''(?i)(dropbox[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{15})['\"]'''
 [[rules]]
 id = "dropbox--api-key"
 description = "Dropbox API secret/key"
 regex = '''(?i)(dropbox[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{15})['\"]'''
 [[rules]]
 id = "dropbox-short-lived-api-token"
 description = "Dropbox short lived API token"
 regex = '''(?i)(dropbox[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](sl\.[a-z0-9\-=_]{135})['\"]'''
 [[rules]]
 id = "dropbox-long-lived-api-token"
 description = "Dropbox long lived API token"
 regex = '''(?i)(dropbox[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"][a-z0-9]{11}(AAAAAAAAAA)[a-z0-9\-_=]{43}['\"]'''
 [[rules]]
 id = "duffel-api-token"
 description = "Duffel API token"
 regex = '''['\"]duffel_(test|live)_(?i)[a-z0-9_-]{43}['\"]'''
 [[rules]]
 id = "dynatrace-api-token"
 description = "Dynatrace API token"
 regex = '''['\"]dt0c01\.(?i)[a-z0-9]{24}\.[a-z0-9]{64}['\"]'''
 [[rules]]
 id = "easypost-api-token"
 description = "EasyPost API token"
 regex = '''['\"]EZAK(?i)[a-z0-9]{54}['\"]'''
 [[rules]]
 id = "easypost-test-api-token"
 description = "EasyPost test API token"
 regex = '''['\"]EZTK(?i)[a-z0-9]{54}['\"]'''
 [[rules]]
 id = "fastly-api-token"
 description = "Fastly API token"
 regex = '''(?i)(fastly[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9\-=_]{32})['\"]'''
 secretGroup = 3
 [[rules]]
 id = "finicity-client-secret"
 description = "Finicity client secret"
 regex = '''(?i)(finicity[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{20})['\"]'''
 secretGroup = 3
 [[rules]]
 id = "finicity-api-token"
 description = "Finicity API token"
 regex = '''(?i)(finicity[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-f0-9]{32})['\"]'''
 secretGroup = 3
 [[rules]]
 id = "flutterwave-public-key"
 description = "Flutterwave public key"
 regex = '''FLWPUBK_TEST-(?i)[a-h0-9]{32}-X'''
 [[rules]]
 id = "flutterwave-secret-key"
 description = "Flutterwave secret key"
 regex = '''FLWSECK_TEST-(?i)[a-h0-9]{32}-X'''
 [[rules]]
 id = "flutterwave-enc-key"
 description = "Flutterwave encrypted key"
 regex = '''FLWSECK_TEST[a-h0-9]{12}'''
 [[rules]]
 id = "frameio-api-token"
 description = "Frame.io API token"
 regex = '''fio-u-(?i)[a-z0-9\-_=]{64}'''
 [[rules]]
 id = "gocardless-api-token"
 description = "GoCardless API token"
 regex = '''['\"]live_(?i)[a-z0-9\-_=]{40}['\"]'''
 [[rules]]
 id = "grafana-api-token"
 description = "Grafana API token"
 regex = '''['\"]eyJrIjoi(?i)[a-z0-9\-_=]{72,92}['\"]'''
 [[rules]]
 id = "hashicorp-tf-api-token"
 description = "HashiCorp Terraform user/org API token"
 regex = '''['\"](?i)[a-z0-9]{14}\.atlasv1\.[a-z0-9\-_=]{60,70}['\"]'''
 [[rules]]
 id = "hubspot-api-token"
 description = "HubSpot API token"
 regex = '''(?i)(hubspot[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-h0-9]{8}-[a-h0-9]{4}-[a-h0-9]{4}-[a-h0-9]{4}-[a-h0-9]{12})['\"]'''
 secretGroup = 3
 [[rules]]
 id = "intercom-api-token"
 description = "Intercom API token"
 regex = '''(?i)(intercom[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9=_]{60})['\"]'''
 secretGroup = 3
 [[rules]]
 id = "intercom-client-secret"
 description = "Intercom client secret/ID"
 regex = '''(?i)(intercom[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-h0-9]{8}-[a-h0-9]{4}-[a-h0-9]{4}-[a-h0-9]{4}-[a-h0-9]{12})['\"]'''
 secretGroup = 3
 [[rules]]
 id = "ionic-api-token"
 description = "Ionic API token"
 regex = '''(?i)(ionic[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](ion_[a-z0-9]{42})['\"]'''
 [[rules]]
 id = "linear-api-token"
 description = "Linear API token"
 regex = '''lin_api_(?i)[a-z0-9]{40}'''
 [[rules]]
 id = "linear-client-secret"
 description = "Linear client secret/ID"
 regex = '''(?i)(linear[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-f0-9]{32})['\"]'''
 secretGroup = 3
 [[rules]]
 id = "lob-api-key"
 description = "Lob API Key"
 regex = '''(?i)(lob[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]((live|test)_[a-f0-9]{35})['\"]'''
 secretGroup = 3
 [[rules]]
 id = "lob-pub-api-key"
 description = "Lob Publishable API Key"
 regex = '''(?i)(lob[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]((test|live)_pub_[a-f0-9]{31})['\"]'''
 secretGroup = 3
 [[rules]]
 id = "mailchimp-api-key"
 description = "Mailchimp API key"
 regex = '''(?i)(mailchimp[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-f0-9]{32}-us20)['\"]'''
 secretGroup = 3
 [[rules]]
 id = "mailgun-private-api-token"
 description = "Mailgun private API token"
 regex = '''(?i)(mailgun[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](key-[a-f0-9]{32})['\"]'''
 secretGroup = 3
 [[rules]]
 id = "mailgun-pub-key"
 description = "Mailgun public validation key"
 regex = '''(?i)(mailgun[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](pubkey-[a-f0-9]{32})['\"]'''
 secretGroup = 3
 [[rules]]
 id = "mailgun-signing-key"
 description = "Mailgun webhook signing key"
 regex = '''(?i)(mailgun[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-h0-9]{32}-[a-h0-9]{8}-[a-h0-9]{8})['\"]'''
 secretGroup = 3
 [[rules]]
 id = "mapbox-api-token"
 description = "Mapbox API token"
 regex = '''(?i)(pk\.[a-z0-9]{60}\.[a-z0-9]{22})'''
 [[rules]]
 id = "messagebird-api-token"
 description = "MessageBird API token"
 regex = '''(?i)(messagebird[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{25})['\"]'''
 secretGroup = 3
 [[rules]]
 id = "messagebird-client-id"
 description = "MessageBird API client ID"
 regex = '''(?i)(messagebird[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-h0-9]{8}-[a-h0-9]{4}-[a-h0-9]{4}-[a-h0-9]{4}-[a-h0-9]{12})['\"]'''
 secretGroup = 3
 [[rules]]
 id = "new-relic-user-api-key"
 description = "New Relic user API Key"
 regex = '''['\"](NRAK-[A-Z0-9]{27})['\"]'''
 [[rules]]
 id = "new-relic-user-api-id"
 description = "New Relic user API ID"
 regex = '''(?i)(newrelic[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([A-Z0-9]{64})['\"]'''
 secretGroup = 3
 [[rules]]
 id = "new-relic-browser-api-token"
 description = "New Relic ingest browser API token"
 regex = '''['\"](NRJS-[a-f0-9]{19})['\"]'''
 [[rules]]
 id = "npm-access-token"
 description = "npm access token"
 regex = '''['\"](npm_(?i)[a-z0-9]{36})['\"]'''
 [[rules]]
 id = "planetscale-password"
 description = "PlanetScale password"
 regex = '''pscale_pw_(?i)[a-z0-9\-_\.]{43}'''
 [[rules]]
 id = "planetscale-api-token"
 description = "PlanetScale API token"
 regex = '''pscale_tkn_(?i)[a-z0-9\-_\.]{43}'''
 [[rules]]
 id = "postman-api-token"
 description = "Postman API token"
 regex = '''PMAK-(?i)[a-f0-9]{24}\-[a-f0-9]{34}'''
 [[rules]]
 id = "pulumi-api-token"
 description = "Pulumi API token"
 regex = '''pul-[a-f0-9]{40}'''
 [[rules]]
 id = "rubygems-api-token"
 description = "Rubygem API token"
 regex = '''rubygems_[a-f0-9]{48}'''
 [[rules]]
 id = "sendgrid-api-token"
 description = "SendGrid API token"
 regex = '''SG\.(?i)[a-z0-9_\-\.]{66}'''
 [[rules]]
 id = "sendinblue-api-token"
 description = "Sendinblue API token"
 regex = '''xkeysib-[a-f0-9]{64}\-(?i)[a-z0-9]{16}'''
 [[rules]]
 id = "shippo-api-token"
 description = "Shippo API token"
 regex = '''shippo_(live|test)_[a-f0-9]{40}'''
 [[rules]]
 id = "linkedin-client-secret"
 description = "LinkedIn Client secret"
 regex = '''(?i)(linkedin[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z]{16})['\"]'''
 secretGroup = 3
 [[rules]]
 id = "linkedin-client-id"
 description = "LinkedIn Client ID"
 regex = '''(?i)(linkedin[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{14})['\"]'''
 secretGroup = 3
 [[rules]]
 id = "twitch-api-token"
 description = "Twitch API token"
 regex = '''(?i)(twitch[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{30})['\"]'''
 secretGroup = 3
 [[rules]]
 id = "typeform-api-token"
 description = "Typeform API token"
 regex = '''(?i)(typeform[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}(tfp_[a-z0-9\-_\.=]{59})'''
 secretGroup = 3
 [[rules]]
 id = "generic-api-key"
 description = "Generic API Key"
 regex = '''(?i)((key|api[^Version]|token|secret|password)[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([0-9a-zA-Z\-_=]{8,64})['\"]'''
 entropy = 3.7
 secretGroup = 4
 [allowlist]
 description = "global allow lists"
 regexes = ['''219-09-9999''', '''078-05-1120''', '''(9[0-9]{2}|666)-\d{2}-\d{4}''', '''RPM-GPG-KEY.*''', '''.*:.*StrelkaHexDump.*''', '''.*:.*PLACEHOLDER.*''', '''ssl_.*password''', '''integration_key\s=\s"so-logs-"''']
 paths = [
    '''gitleaks.toml''',
    '''(.*?)(jpg|gif|doc|pdf|bin|svg|socket)$''',
    '''(go.mod|go.sum)$''',
    '''salt/nginx/files/enterprise-attack.json''',
    '''(.*?)whl$'''
 ]
@@ -2,13 +2,11 @@ body:
  - type: markdown
    attributes:
      value: |
        ⚠️ This category is solely for conversations related to Security Onion 2.4 ⚠️
        If your organization needs more immediate, enterprise grade professional support, with one-on-one virtual meetings and screensharing, contact us via our website: https://securityonion.com/support
  - type: dropdown
    attributes:
      label: Version
-      description: Which version of Security Onion 2.4.x are you asking about?
+      description: Which version of Security Onion are you asking about?
      options:
        -
        - 2.4.10
@@ -31,6 +29,11 @@ body:
        - 2.4.160
        - 2.4.170
        - 2.4.180
        - 2.4.190
        - 2.4.200
        - 2.4.201
        - 2.4.210
        - 2.4.211
        - Other (please provide detail below)
    validations:
      required: true
@@ -92,7 +95,7 @@ body:
    attributes:
      label: Hardware Specs
      description: >
-        Does your hardware meet or exceed the minimum requirements for your installation type as shown at https://docs.securityonion.net/en/2.4/hardware.html?
+        Does your hardware meet or exceed the minimum requirements for your installation type as shown at https://securityonion.net/docs/hardware?
      options:
        -
        - Meets minimum requirements
@@ -0,0 +1,178 @@
 body:
  - type: markdown
    attributes:
      value: |
        If your organization needs more immediate, enterprise grade professional support, with one-on-one virtual meetings and screensharing, contact us via our website: https://securityonion.com/support
  - type: dropdown
    attributes:
      label: Version
      description: Which version of Security Onion are you asking about?
      options:
        -
        - 3.0.0
        - 3.1.0
        - Other (please provide detail below)
    validations:
      required: true
  - type: dropdown
    attributes:
      label: Installation Method
      description: How did you install Security Onion?
      options:
        -
        - Security Onion ISO image
        - Cloud image (Amazon, Azure, Google)
        - Network installation on Oracle 9 (unsupported)
        - Other (please provide detail below)
    validations:
      required: true
  - type: dropdown
    attributes:
      label: Description
      description: >
        Is this discussion about installation, configuration, upgrading, or other?
      options:
        -
        - installation
        - configuration
        - upgrading
        - other (please provide detail below)
    validations:
      required: true
  - type: dropdown
    attributes:
      label: Installation Type
      description: >
        When you installed, did you choose Import, Eval, Standalone, Distributed, or something else?
      options:
        -
        - Import
        - Eval
        - Standalone
        - Distributed
        - other (please provide detail below)
    validations:
      required: true
  - type: dropdown
    attributes:
      label: Location
      description: >
        Is this deployment in the cloud, on-prem with Internet access, or airgap?
      options:
        -
        - cloud
        - on-prem with Internet access
        - airgap
        - other (please provide detail below)
    validations:
      required: true
  - type: dropdown
    attributes:
      label: Hardware Specs
      description: >
        Does your hardware meet or exceed the minimum requirements for your installation type as shown at https://securityonion.net/docs/hardware?
      options:
        -
        - Meets minimum requirements
        - Exceeds minimum requirements
        - Does not meet minimum requirements
        - other (please provide detail below)
    validations:
      required: true
  - type: input
    attributes:
      label: CPU
      description: How many CPU cores do you have?
    validations:
      required: true
  - type: input
    attributes:
      label: RAM
      description: How much RAM do you have?
    validations:
      required: true
  - type: input
    attributes:
      label: Storage for /
      description: How much storage do you have for the / partition?
    validations:
      required: true
  - type: input
    attributes:
      label: Storage for /nsm
      description: How much storage do you have for the /nsm partition?
    validations:
      required: true
  - type: dropdown
    attributes:
      label: Network Traffic Collection
      description: >
        Are you collecting network traffic from a tap or span port?
      options:
        -
        - tap
        - span port
        - other (please provide detail below)
    validations:
      required: true
  - type: dropdown
    attributes:
      label: Network Traffic Speeds
      description: >
        How much network traffic are you monitoring?
      options:
        -
        - Less than 1Gbps
        - 1Gbps to 10Gbps
        - more than 10Gbps
    validations:
      required: true
  - type: dropdown
    attributes:
      label: Status
      description: >
        Does SOC Grid show all services on all nodes as running OK?
      options:
        -
        - Yes, all services on all nodes are running OK
        - No, one or more services are failed (please provide detail below)
    validations:
      required: true
  - type: dropdown
    attributes:
      label: Salt Status
      description: >
        Do you get any failures when you run "sudo salt-call state.highstate"?
      options:
        -
        - Yes, there are salt failures (please provide detail below)
        - No, there are no failures
    validations:
      required: true
  - type: dropdown
    attributes:
      label: Logs
      description: >
        Are there any additional clues in /opt/so/log/?
      options:
        -
        - Yes, there are additional clues in /opt/so/log/ (please provide detail below)
        - No, there are no additional clues
    validations:
      required: true
  - type: textarea
    attributes:
      label: Detail
      description: Please read our discussion guidelines at https://github.com/Security-Onion-Solutions/securityonion/discussions/1720 and then provide detailed information to help us help you.
      placeholder: |-
        STOP! Before typing, please read our discussion guidelines at https://github.com/Security-Onion-Solutions/securityonion/discussions/1720 in their entirety!
        If your organization needs more immediate, enterprise grade professional support, with one-on-one virtual meetings and screensharing, contact us via our website: https://securityonion.com/support
    validations:
      required: true
  - type: checkboxes
    attributes:
      label: Guidelines
      options:
        - label: I have read the discussion guidelines at https://github.com/Security-Onion-Solutions/securityonion/discussions/1720 and assert that I have followed the guidelines.
          required: true
@@ -0,0 +1,22 @@
 ## Description
 <!-- 
 Explain the purpose of the pull request. Be brief or detailed depending on the scope of the changes.
 -->
 ## Related Issues
 <!-- 
 Optionally, list any related issues that this pull request addresses.
 -->
 ## Checklist
 - [ ] I have read and followed the [CONTRIBUTING.md](https://github.com/Security-Onion-Solutions/securityonion/blob/3/main/CONTRIBUTING.md) file.
 - [ ] I have read and agree to the terms of the [Contributor License Agreement](https://securityonionsolutions.com/cla)
 ## Questions or Comments
 <!--
 If you have any questions or comments about this pull request, add them here.
 -->
@@ -1,24 +0,0 @@
 name: contrib
 on:
  issue_comment:
    types: [created]
  pull_request_target:
    types: [opened,closed,synchronize]
 jobs:
  CLAssistant:
    runs-on: ubuntu-latest
    steps:
      - name: "Contributor Check"
        if: (github.event.comment.body == 'recheck' || github.event.comment.body == 'I have read the CLA Document and I hereby sign the CLA') || github.event_name == 'pull_request_target'
        uses: cla-assistant/github-action@v2.3.1
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          PERSONAL_ACCESS_TOKEN : ${{ secrets.PERSONAL_ACCESS_TOKEN }}
        with:
          path-to-signatures: 'signatures_v1.json'
          path-to-document: 'https://securityonionsolutions.com/cla' 
          allowlist: dependabot[bot],jertel,dougburks,TOoSmOotH,defensivedepth,m0duspwnens
          remote-organization-name: Security-Onion-Solutions
          remote-repository-name: licensing
@@ -1,17 +0,0 @@
 name: leak-test
 on: [pull_request]
 jobs:
  build:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v2
      with:
        fetch-depth: '0'
    - name: Gitleaks
      uses: gitleaks/gitleaks-action@v1.6.0
      with:
        config-path: .github/.gitleaks.toml
@@ -4,7 +4,7 @@ on:
  pull_request:
    paths:
      - "salt/sensoroni/files/analyzers/**"
-      - "salt/manager/tools/sbin"
+      - "salt/manager/tools/sbin/**"
 jobs:
  build:
@@ -13,7 +13,7 @@ jobs:
    strategy:
      fail-fast: false
      matrix:
-        python-version: ["3.13"]
+        python-version: ["3.14"]
        python-code-path: ["salt/sensoroni/files/analyzers", "salt/manager/tools/sbin"]
    steps:
@@ -23,7 +23,7 @@
 * Link the PR to the related issue, either using [keywords](https://docs.github.com/en/issues/tracking-your-work-with-issues/creating-issues/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword) in the PR description, or [manually](https://docs.github.com/en/issues/tracking-your-work-with-issues/creating-issues/linking-a-pull-request-to-an-issue#manually-linking-a-pull-request-to-an-issue).
-* **Pull requests should be opened against the `dev` branch of this repo**, and should clearly describe the problem and solution.
+* **Pull requests should be opened against the current `?/dev` branch of this repo**, and should clearly describe the problem and solution.
 * Be sure you have tested your changes and are confident they will not break other parts of the product.
@@ -1,46 +1,46 @@
-### 2.4.180-20250916 ISO image released on 2025/09/17
+### 3.0.0-20260331 ISO image released on 2026/03/31
 ### Download and Verify
-2.4.180-20250916 ISO image:  
+3.0.0-20260331 ISO image:  
-https://download.securityonion.net/file/securityonion/securityonion-2.4.180-20250916.iso
+https://download.securityonion.net/file/securityonion/securityonion-3.0.0-20260331.iso
-MD5: DE93880E38DE4BE45D05A41E1745CB1F  
+MD5: ECD318A1662A6FDE0EF213F5A9BD4B07  
-SHA1: AEA6948911E50A4A38E8729E0E965C565402E3FC  
+SHA1: E55BE314440CCF3392DC0B06BC5E270B43176D9C  
-SHA256: C9BD8CA071E43B048ABF9ED145B87935CB1D4BB839B2244A06FAD1BBA8EAC84A  
+SHA256: 7FC47405E335CBE5C2B6C51FE7AC60248F35CBE504907B8B5A33822B23F8F4D5  
 Signature for ISO image:  
-https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.180-20250916.iso.sig
+https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.0.0-20260331.iso.sig
 Signing key:  
-https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.4/main/KEYS  
+https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/3/main/KEYS  
 For example, here are the steps you can use on most Linux distributions to download and verify our Security Onion ISO image.
 Download and import the signing key:  
 ```
-wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.4/main/KEYS -O - | gpg --import -  
+wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/3/main/KEYS -O - | gpg --import -  
 ```
 Download the signature file for the ISO:  
 ```
-wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.180-20250916.iso.sig
+wget https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.0.0-20260331.iso.sig
 ```
 Download the ISO image:  
 ```
-wget https://download.securityonion.net/file/securityonion/securityonion-2.4.180-20250916.iso
+wget https://download.securityonion.net/file/securityonion/securityonion-3.0.0-20260331.iso
 ```
 Verify the downloaded ISO image using the signature file:  
 ```
-gpg --verify securityonion-2.4.180-20250916.iso.sig securityonion-2.4.180-20250916.iso
+gpg --verify securityonion-3.0.0-20260331.iso.sig securityonion-3.0.0-20260331.iso
 ```
 The output should show "Good signature" and the Primary key fingerprint should match what's shown below:
 ```
-gpg: Signature made Tue 16 Sep 2025 06:30:19 PM EDT using RSA key ID FE507013
+gpg: Signature made Mon 30 Mar 2026 06:22:14 PM EDT using RSA key ID FE507013
 gpg: Good signature from "Security Onion Solutions, LLC <info@securityonionsolutions.com>"
 gpg: WARNING: This key is not certified with a trusted signature!
 gpg:          There is no indication that the signature belongs to the owner.
@@ -50,4 +50,4 @@ Primary key fingerprint: C804 A93D 36BE 0C73 3EA1  9644 7C10 60B7 FE50 7013
 If it fails to verify, try downloading again. If it still fails to verify, try downloading from another computer or another network.
 Once you've verified the ISO image, you're ready to proceed to our Installation guide:  
-https://docs.securityonion.net/en/2.4/installation.html
+https://securityonion.net/docs/installation
@@ -1,50 +1,58 @@
-## Security Onion 2.4
+<p align="center">
  <img src="https://securityonionsolutions.com/logo/logo-so-onion-dark.svg" width="400" alt="Security Onion Logo">
 </p>
-Security Onion 2.4 is here!
+# Security Onion
-## Screenshots
+Security Onion is a free and open Linux distribution for threat hunting, enterprise security monitoring, and log management. It includes a comprehensive suite of tools designed to work together to provide visibility into your network and host activity.
-Alerts
+## ✨ Features
 ![Alerts](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/50_alerts.png)
-Dashboards
+Security Onion includes everything you need to monitor your network and host systems:
 ![Dashboards](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/53_dashboards.png)
-Hunt
+*   **Security Onion Console (SOC)**: A unified web interface for analyzing security events and managing your grid.
-![Hunt](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/56_hunt.png)
+*   **Elastic Stack**: Powerful search backed by Elasticsearch.
 *   **Intrusion Detection**: Network-based IDS with Suricata and host-based monitoring with Elastic Fleet.
 *   **Network Metadata**: Detailed network metadata generated by Zeek or Suricata.
 *   **Full Packet Capture**: Retain and analyze raw network traffic with Suricata PCAP.
-Detections
+## ⭐ Security Onion Pro
 ![Detections](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/57_detections.png)
-PCAP
+For organizations and enterprises requiring advanced capabilities, **Security Onion Pro** offers additional features designed for scale and efficiency:
 ![PCAP](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/62_pcap.png)
-Grid
+*   **Onion AI**: Leverage powerful AI-driven insights to accelerate your analysis and investigations.
-![Grid](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/75_grid.png)
+*   **Enterprise Features**: Enhanced tools and integrations tailored for enterprise-grade security operations.
-Config
+For more information, visit the [Security Onion Pro](https://securityonionsolutions.com/pro) page.
 ![Config](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/87_config.png)
-### Release Notes
+## ☁️ Cloud Deployment
-https://docs.securityonion.net/en/2.4/release-notes.html
+Security Onion is available and ready to deploy in the **AWS**, **Azure**, and **Google Cloud (GCP)** marketplaces.
-### Requirements
+## 🚀 Getting Started
-https://docs.securityonion.net/en/2.4/hardware.html
+| Goal | Resource |
 | :--- | :--- |
 | **Download** | [Security Onion ISO](https://securityonion.net/docs/download) |
 | **Requirements** | [Hardware Guide](https://securityonion.net/docs/hardware) |
 | **Install** | [Installation Instructions](https://securityonion.net/docs/installation) |
 | **What's New** | [Release Notes](https://securityonion.net/docs/release-notes) |
-### Download
+## 📖 Documentation & Support
-https://docs.securityonion.net/en/2.4/download.html
+For more detailed information, please visit our [Documentation](https://docs.securityonion.net).
-### Installation
+*   **FAQ**: [Frequently Asked Questions](https://securityonion.net/docs/faq)
 *   **Community**: [Discussions & Support](https://securityonion.net/docs/community-support)
 *   **Training**: [Official Training](https://securityonion.net/training)
-https://docs.securityonion.net/en/2.4/installation.html
+## 🤝 Contributing
-### FAQ
+We welcome contributions! Please see our [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines on how to get involved.
-https://docs.securityonion.net/en/2.4/faq.html
+## 🛡️ License
-### Feedback
+Security Onion is licensed under the terms of the license found in the [LICENSE](LICENSE) file.
-https://docs.securityonion.net/en/2.4/community-support.html
+---
 *Built with 🧅 by Security Onion Solutions.*
@@ -4,6 +4,7 @@
 | Version | Supported          |
 | ------- | ------------------ |
 | 3.x     | :white_check_mark: |
 | 2.4.x   | :white_check_mark: |
 | 2.3.x   | :x:                |
 | 16.04.x | :x:                |
@@ -1 +1 @@
-2.4.180
+3.1.0
@@ -0,0 +1,2 @@
 ca:
  server:
@@ -1,2 +0,0 @@
 elasticsearch:
  index_settings:
@@ -1,5 +1,6 @@
 base:
  '*':
    - ca
    - global.soc_global
    - global.adv_global
    - docker.soc_docker
@@ -43,8 +44,6 @@ base:
    - secrets
    - manager.soc_manager
    - manager.adv_manager
    - idstools.soc_idstools
    - idstools.adv_idstools
    - logstash.nodes
    - logstash.soc_logstash
    - logstash.adv_logstash
@@ -88,8 +87,6 @@ base:
    - zeek.adv_zeek
    - bpf.soc_bpf
    - bpf.adv_bpf
    - pcap.soc_pcap
    - pcap.adv_pcap
    - suricata.soc_suricata
    - suricata.adv_suricata
    - minions.{{ grains.id }}
@@ -100,7 +97,6 @@ base:
    - node_data.ips
    - secrets
    - healthcheck.eval
    - elasticsearch.index_templates
    {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
    - elasticsearch.auth
    {% endif %}
@@ -117,8 +113,6 @@ base:
    - elastalert.adv_elastalert
    - manager.soc_manager
    - manager.adv_manager
    - idstools.soc_idstools
    - idstools.adv_idstools
    - soc.soc_soc
    - soc.adv_soc
    - kibana.soc_kibana
@@ -137,8 +131,6 @@ base:
    - zeek.adv_zeek
    - bpf.soc_bpf
    - bpf.adv_bpf
    - pcap.soc_pcap
    - pcap.adv_pcap
    - suricata.soc_suricata
    - suricata.adv_suricata
    - minions.{{ grains.id }}
@@ -149,7 +141,6 @@ base:
    - logstash.nodes
    - logstash.soc_logstash
    - logstash.adv_logstash
    - elasticsearch.index_templates
    {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
    - elasticsearch.auth
    {% endif %}
@@ -158,8 +149,6 @@ base:
    {% endif %}
    - secrets
    - healthcheck.standalone
    - idstools.soc_idstools
    - idstools.adv_idstools
    - kratos.soc_kratos
    - kratos.adv_kratos
    - hydra.soc_hydra
@@ -190,8 +179,6 @@ base:
    - zeek.adv_zeek
    - bpf.soc_bpf
    - bpf.adv_bpf
    - pcap.soc_pcap
    - pcap.adv_pcap
    - suricata.soc_suricata
    - suricata.adv_suricata
    - minions.{{ grains.id }}
@@ -214,8 +201,6 @@ base:
    - zeek.adv_zeek
    - bpf.soc_bpf
    - bpf.adv_bpf
    - pcap.soc_pcap
    - pcap.adv_pcap
    - suricata.soc_suricata
    - suricata.adv_suricata
    - strelka.soc_strelka
@@ -269,7 +254,6 @@ base:
  '*_import':
    - node_data.ips
    - secrets
    - elasticsearch.index_templates
    {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
    - elasticsearch.auth
    {% endif %}
@@ -302,8 +286,6 @@ base:
    - zeek.adv_zeek
    - bpf.soc_bpf
    - bpf.adv_bpf
    - pcap.soc_pcap
    - pcap.adv_pcap
    - suricata.soc_suricata
    - suricata.adv_suricata
    - strelka.soc_strelka
@@ -0,0 +1,91 @@
 #!/opt/saltstack/salt/bin/python3
 # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
 # or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 #
 # Note: Per the Elastic License 2.0, the second limitation states:
 #
 #   "You may not move, change, disable, or circumvent the license key functionality
 #    in the software, and you may not remove or obscure any functionality in the
 #    software that is protected by the license key."
 """
 Salt execution module for hypervisor operations.
 This module provides functions for managing hypervisor configurations,
 including VM file management.
 """
 import json
 import logging
 import os
 log = logging.getLogger(__name__)
 __virtualname__ = 'hypervisor'
 def __virtual__():
    """
    Only load this module if we're on a system that can manage hypervisors.
    """
    return __virtualname__
 def remove_vm_from_vms_file(vms_file_path, vm_hostname, vm_role):
    """
    Remove a VM entry from the hypervisorVMs file.
    Args:
        vms_file_path (str): Path to the hypervisorVMs file
        vm_hostname (str): Hostname of the VM to remove (without role suffix)
        vm_role (str): Role of the VM
    Returns:
        dict: Result dictionary with success status and message
    CLI Example:
        salt '*' hypervisor.remove_vm_from_vms_file /opt/so/saltstack/local/salt/hypervisor/hosts/hypervisor1VMs node1 nsm
    """
    try:
        # Check if file exists
        if not os.path.exists(vms_file_path):
            msg = f"VMs file not found: {vms_file_path}"
            log.error(msg)
            return {'result': False, 'comment': msg}
        # Read current VMs
        with open(vms_file_path, 'r') as f:
            content = f.read().strip()
            vms = json.loads(content) if content else []
        # Find and remove the VM entry
        original_count = len(vms)
        vms = [vm for vm in vms if not (vm.get('hostname') == vm_hostname and vm.get('role') == vm_role)]
        if len(vms) < original_count:
            # VM was found and removed, write back to file
            with open(vms_file_path, 'w') as f:
                json.dump(vms, f, indent=2)
            # Set socore:socore ownership (939:939)
            os.chown(vms_file_path, 939, 939)
            msg = f"Removed VM {vm_hostname}_{vm_role} from {vms_file_path}"
            log.info(msg)
            return {'result': True, 'comment': msg}
        else:
            msg = f"VM {vm_hostname}_{vm_role} not found in {vms_file_path}"
            log.warning(msg)
            return {'result': False, 'comment': msg}
    except json.JSONDecodeError as e:
        msg = f"Failed to parse JSON in {vms_file_path}: {str(e)}"
        log.error(msg)
        return {'result': False, 'comment': msg}
    except Exception as e:
        msg = f"Failed to remove VM {vm_hostname}_{vm_role} from {vms_file_path}: {str(e)}"
        log.error(msg)
        return {'result': False, 'comment': msg}
@@ -1,24 +1,14 @@
 from os import path
 import subprocess
 def check():
  osfam = __grains__['os_family']
  retval = 'False'
-  if osfam == 'Debian':
+  cmd = 'needs-restarting -r > /dev/null 2>&1'
    if path.exists('/var/run/reboot-required'):
      retval = 'True'
-  elif osfam == 'RedHat':
+  try:
-    cmd = 'needs-restarting -r > /dev/null 2>&1'
+    needs_restarting = subprocess.check_call(cmd, shell=True)
-    
+  except subprocess.CalledProcessError:
-    try:
+    retval = 'True'
      needs_restarting = subprocess.check_call(cmd, shell=True)
    except subprocess.CalledProcessError:
      retval = 'True'
  else:
    retval = 'Unsupported OS: %s' % os
  return retval
@@ -7,12 +7,14 @@
 """
 Salt module for managing QCOW2 image configurations and VM hardware settings. This module provides functions
-for modifying network configurations within QCOW2 images and adjusting virtual machine hardware settings.
+for modifying network configurations within QCOW2 images, adjusting virtual machine hardware settings, and
-It serves as a Salt interface to the so-qcow2-modify-network and so-kvm-modify-hardware scripts.
+creating virtual storage volumes. It serves as a Salt interface to the so-qcow2-modify-network,
 so-kvm-modify-hardware, and so-kvm-create-volume scripts.
-The module offers two main capabilities:
+The module offers three main capabilities:
 1. Network Configuration: Modify network settings (DHCP/static IP) within QCOW2 images
 2. Hardware Configuration: Adjust VM hardware settings (CPU, memory, PCI passthrough)
 3. Volume Management: Create and attach virtual storage volumes for NSM data
 This module is intended to work with Security Onion's virtualization infrastructure and is typically
 used in conjunction with salt-cloud for VM provisioning and management.
@@ -244,3 +246,90 @@ def modify_hardware_config(vm_name, cpu=None, memory=None, pci=None, start=False
    except Exception as e:
        log.error('qcow2 module: An error occurred while executing the script: {}'.format(e))
        raise
 def create_volume_config(vm_name, size_gb, start=False):
    '''
    Usage:
        salt '*' qcow2.create_volume_config vm_name=<name> size_gb=<size> [start=<bool>]
    Options:
        vm_name
            Name of the virtual machine to attach the volume to
        size_gb
            Volume size in GB (positive integer)
            This determines the capacity of the virtual storage volume
        start
            Boolean flag to start the VM after volume creation
            Optional - defaults to False
    Examples:
        1. **Create 500GB Volume:**
            ```bash
            salt '*' qcow2.create_volume_config vm_name='sensor1_sensor' size_gb=500
            ```
            This creates a 500GB virtual volume for NSM storage
        2. **Create 1TB Volume and Start VM:**
            ```bash
            salt '*' qcow2.create_volume_config vm_name='sensor1_sensor' size_gb=1000 start=True
            ```
            This creates a 1TB volume and starts the VM after attachment
    Notes:
        - VM must be stopped before volume creation
        - Volume is created as a qcow2 image and attached to the VM
        - This is an alternative to disk passthrough via modify_hardware_config
        - Volume is automatically attached to the VM's libvirt configuration
        - Requires so-kvm-create-volume script to be installed
        - Volume files are stored in the hypervisor's VM storage directory
    Description:
        This function creates and attaches a virtual storage volume to a KVM virtual machine
        using the so-kvm-create-volume script. It creates a qcow2 disk image of the specified
        size and attaches it to the VM for NSM (Network Security Monitoring) storage purposes.
        This provides an alternative to physical disk passthrough, allowing flexible storage
        allocation without requiring dedicated hardware. The VM can optionally be started
        after the volume is successfully created and attached.
    Exit Codes:
        0: Success
        1: Invalid parameters
        2: VM state error (running when should be stopped)
        3: Volume creation error
        4: System command error
        255: Unexpected error
    Logging:
        - All operations are logged to the salt minion log
        - Log entries are prefixed with 'qcow2 module:'
        - Volume creation and attachment operations are logged
        - Errors include detailed messages and stack traces
        - Final status of volume creation is logged
    '''
    # Validate size_gb parameter
    if not isinstance(size_gb, int) or size_gb <= 0:
        raise ValueError('size_gb must be a positive integer.')
    cmd = ['/usr/sbin/so-kvm-create-volume', '-v', vm_name, '-s', str(size_gb)]
    if start:
        cmd.append('-S')
    log.info('qcow2 module: Executing command: {}'.format(' '.join(shlex.quote(arg) for arg in cmd)))
    try:
        result = subprocess.run(cmd, capture_output=True, text=True, check=False)
        ret = {
            'retcode': result.returncode,
            'stdout': result.stdout,
            'stderr': result.stderr
        }
        if result.returncode != 0:
            log.error('qcow2 module: Script execution failed with return code {}: {}'.format(result.returncode, result.stderr))
        else:
            log.info('qcow2 module: Script executed successfully.')
        return ret
    except Exception as e:
        log.error('qcow2 module: An error occurred while executing the script: {}'.format(e))
        raise
@@ -172,7 +172,15 @@ MANAGER_HOSTNAME = socket.gethostname()
 def _download_image():
    """
-    Download and validate the Oracle Linux KVM image.
+    Download and validate the Oracle Linux KVM image with retry logic and progress monitoring.
    Features:
    - Detects stalled downloads (no progress for 30 seconds)
    - Retries up to 3 times on failure
    - Connection timeout of 30 seconds
    - Read timeout of 60 seconds
    - Cleans up partial downloads on failure
    Returns:
        bool: True if successful or file exists with valid checksum, False on error
    """
@@ -185,45 +193,107 @@ def _download_image():
            os.unlink(IMAGE_PATH)
    log.info("Starting image download process")
    # Retry configuration
    max_attempts = 3
    retry_delay = 5  # seconds to wait between retry attempts
    stall_timeout = 30  # seconds without progress before considering download stalled
    connection_timeout = 30  # seconds to establish connection
    read_timeout = 60  # seconds to wait for data chunks
    for attempt in range(1, max_attempts + 1):
        log.info("Download attempt %d of %d", attempt, max_attempts)
        try:
            # Download file with timeouts
            log.info("Downloading Oracle Linux KVM image from %s to %s", IMAGE_URL, IMAGE_PATH)
            response = requests.get(
                IMAGE_URL,
                stream=True,
                timeout=(connection_timeout, read_timeout)
            )
            response.raise_for_status()
-    try:
+            # Get total file size for progress tracking
-        # Download file
+            total_size = int(response.headers.get('content-length', 0))
-        log.info("Downloading Oracle Linux KVM image from %s to %s", IMAGE_URL, IMAGE_PATH)
+            downloaded_size = 0
-        response = requests.get(IMAGE_URL, stream=True)
+            last_log_time = 0
-        response.raise_for_status()
+            last_progress_time = time.time()
            last_downloaded_size = 0
-        # Get total file size for progress tracking
+            # Save file with progress logging and stall detection
-        total_size = int(response.headers.get('content-length', 0))
+            with salt.utils.files.fopen(IMAGE_PATH, 'wb') as f:
-        downloaded_size = 0
+                for chunk in response.iter_content(chunk_size=8192):
-        last_log_time = 0
+                    if chunk:  # filter out keep-alive new chunks
                        f.write(chunk)
                        downloaded_size += len(chunk)
                        current_time = time.time()
                        # Check for stalled download
                        if downloaded_size > last_downloaded_size:
                            # Progress made, reset stall timer
                            last_progress_time = current_time
                            last_downloaded_size = downloaded_size
                        elif current_time - last_progress_time > stall_timeout:
                            # No progress for stall_timeout seconds
                            raise Exception(
                                f"Download stalled: no progress for {stall_timeout} seconds "
                                f"at {downloaded_size}/{total_size} bytes"
                            )
                        # Log progress every second
                        if current_time - last_log_time >= 1:
                            progress = (downloaded_size / total_size) * 100 if total_size > 0 else 0
                            log.info("Progress - %.1f%% (%d/%d bytes)",
                                    progress, downloaded_size, total_size)
                            last_log_time = current_time
-        # Save file with progress logging
+            # Validate downloaded file
-        with salt.utils.files.fopen(IMAGE_PATH, 'wb') as f:
+            log.info("Download complete, validating checksum...")
-            for chunk in response.iter_content(chunk_size=8192):
+            if not _validate_image_checksum(IMAGE_PATH, IMAGE_SHA256):
-                f.write(chunk)
+                log.error("Checksum validation failed on attempt %d", attempt)
-                downloaded_size += len(chunk)
+                os.unlink(IMAGE_PATH)
                if attempt < max_attempts:
                    log.info("Will retry download...")
                    continue
                else:
                    log.error("All download attempts failed due to checksum mismatch")
                    return False
            log.info("Successfully downloaded and validated Oracle Linux KVM image")
            return True
        except requests.exceptions.Timeout as e:
            log.error("Download attempt %d failed: Timeout - %s", attempt, str(e))
            if os.path.exists(IMAGE_PATH):
                os.unlink(IMAGE_PATH)
            if attempt < max_attempts:
                log.info("Will retry download in %d seconds...", retry_delay)
                time.sleep(retry_delay)
            else:
                log.error("All download attempts failed due to timeout")
-                # Log progress every second
+        except requests.exceptions.RequestException as e:
-                current_time = time.time()
+            log.error("Download attempt %d failed: Network error - %s", attempt, str(e))
-                if current_time - last_log_time >= 1:
+            if os.path.exists(IMAGE_PATH):
-                    progress = (downloaded_size / total_size) * 100 if total_size > 0 else 0
+                os.unlink(IMAGE_PATH)
-                    log.info("Progress - %.1f%% (%d/%d bytes)", 
+            if attempt < max_attempts:
-                            progress, downloaded_size, total_size)
+                log.info("Will retry download in %d seconds...", retry_delay)
-                    last_log_time = current_time
+                time.sleep(retry_delay)
-
+            else:
-        # Validate downloaded file
+                log.error("All download attempts failed due to network errors")
-        if not _validate_image_checksum(IMAGE_PATH, IMAGE_SHA256):
+                
-            os.unlink(IMAGE_PATH)
+        except Exception as e:
-            return False
+            log.error("Download attempt %d failed: %s", attempt, str(e))
-
+            if os.path.exists(IMAGE_PATH):
-        log.info("Successfully downloaded and validated Oracle Linux KVM image")
+                os.unlink(IMAGE_PATH)
-        return True
+            if attempt < max_attempts:
-
+                log.info("Will retry download in %d seconds...", retry_delay)
-    except Exception as e:
+                time.sleep(retry_delay)
-        log.error("Error downloading hypervisor image: %s", str(e))
+            else:
-        if os.path.exists(IMAGE_PATH):
+                log.error("All download attempts failed")
-            os.unlink(IMAGE_PATH)
+    
-        return False
+    return False
 def _check_ssh_keys_exist():
    """
@@ -419,25 +489,28 @@ def _ensure_hypervisor_host_dir(minion_id: str = None):
        log.error(f"Error creating hypervisor host directory: {str(e)}")
        return False
-def _apply_dyanno_hypervisor_state():
+def _apply_dyanno_hypervisor_state(status):
    """
    Apply the soc.dyanno.hypervisor state on the salt master.
    This function applies the soc.dyanno.hypervisor state on the salt master
    to update the hypervisor annotation and ensure all hypervisor host directories exist.
    Args:
        status: Status passed to the hypervisor annotation state
    Returns:
        bool: True if state was applied successfully, False otherwise
    """
    try:
-        log.info("Applying soc.dyanno.hypervisor state on salt master")
+        log.info(f"Applying soc.dyanno.hypervisor state on salt master with status: {status}")
        # Initialize the LocalClient
        local = salt.client.LocalClient()
        # Target the salt master to apply the soc.dyanno.hypervisor state
        target = MANAGER_HOSTNAME + '_*'
-        state_result = local.cmd(target, 'state.apply', ['soc.dyanno.hypervisor', "pillar={'baseDomain': {'status': 'PreInit'}}", 'concurrent=True'], tgt_type='glob')
+        state_result = local.cmd(target, 'state.apply', ['soc.dyanno.hypervisor', f"pillar={{'baseDomain': {{'status': '{status}'}}}}", 'concurrent=True'], tgt_type='glob')
        log.debug(f"state_result: {state_result}")
        # Check if state was applied successfully
        if state_result:
@@ -454,17 +527,17 @@ def _apply_dyanno_hypervisor_state():
                        success = False
            if success:
-                log.info("Successfully applied soc.dyanno.hypervisor state")
+                log.info(f"Successfully applied soc.dyanno.hypervisor state with status: {status}")
                return True
            else:
-                log.error("Failed to apply soc.dyanno.hypervisor state")
+                log.error(f"Failed to apply soc.dyanno.hypervisor state with status: {status}")
                return False
        else:
-            log.error("No response from salt master when applying soc.dyanno.hypervisor state")
+            log.error(f"No response from salt master when applying soc.dyanno.hypervisor state with status: {status}")
            return False
    except Exception as e:
-        log.error(f"Error applying soc.dyanno.hypervisor state: {str(e)}")
+        log.error(f"Error applying soc.dyanno.hypervisor state with status: {status}: {str(e)}")
        return False
 def _apply_cloud_config_state():
@@ -598,11 +671,6 @@ def setup_environment(vm_name: str = 'sool9', disk_size: str = '220G', minion_id
        log.warning("Failed to apply salt.cloud.config state, continuing with setup")
        # We don't return an error here as we want to continue with the setup process
    # Apply the soc.dyanno.hypervisor state on the salt master
    if not _apply_dyanno_hypervisor_state():
        log.warning("Failed to apply soc.dyanno.hypervisor state, continuing with setup")
        # We don't return an error here as we want to continue with the setup process
    log.info("Starting setup_environment in setup_hypervisor runner")
    # Check if environment is already set up
@@ -616,9 +684,12 @@ def setup_environment(vm_name: str = 'sool9', disk_size: str = '220G', minion_id
    # Handle image setup if needed
    if not image_valid:
        _apply_dyanno_hypervisor_state('ImageDownloadStart')
        log.info("Starting image download/validation process")
        if not _download_image():
            log.error("Image download failed")
            # Update hypervisor annotation with failure status
            _apply_dyanno_hypervisor_state('ImageDownloadFailed')
            return {
                'success': False,
                'error': 'Image download failed',
@@ -631,6 +702,8 @@ def setup_environment(vm_name: str = 'sool9', disk_size: str = '220G', minion_id
        log.info("Setting up SSH keys")
        if not _setup_ssh_keys():
            log.error("SSH key setup failed")
            # Update hypervisor annotation with failure status
            _apply_dyanno_hypervisor_state('SSHKeySetupFailed')
            return {
                'success': False,
                'error': 'SSH key setup failed',
@@ -655,6 +728,12 @@ def setup_environment(vm_name: str = 'sool9', disk_size: str = '220G', minion_id
    success = vm_result.get('success', False)
    log.info("Setup environment completed with status: %s", "SUCCESS" if success else "FAILED")
    # Update hypervisor annotation with success status
    if success:
        _apply_dyanno_hypervisor_state('PreInit')
    else:
        _apply_dyanno_hypervisor_state('SetupFailed')
    # If setup was successful and we have a minion_id, run highstate
    if success and minion_id:
        log.info("Running highstate on hypervisor %s", minion_id)
@@ -15,11 +15,7 @@
    'salt.minion-check',
    'sensoroni',
    'salt.lasthighstate',
-    'salt.minion'
+    'salt.minion',
 ] %}
 {% set ssl_states = [
    'ssl',
    'telegraf',
    'firewall',
    'schedule',
@@ -28,7 +24,7 @@
 {% set manager_states = [
    'salt.master',
-    'ca',
+    'ca.server',
    'registry',
    'manager',
    'nginx',
@@ -38,13 +34,10 @@
    'hydra',
    'elasticfleet',
    'elastic-fleet-package-registry',
    'idstools',
    'suricata.manager',
    'utility'
 ] %}
 {% set sensor_states = [
    'pcap',
    'suricata',
    'healthcheck',
    'tcpreplay',
@@ -77,28 +70,24 @@
    {# Map role-specific states #}
    {% set role_states = {
        'so-eval': (
            ssl_states +
            manager_states +
            sensor_states +
-            elastic_stack_states | reject('equalto', 'logstash') | list
+            elastic_stack_states | reject('equalto', 'logstash') | list +
            ['logstash.ssl']
        ),
        'so-heavynode': (
            ssl_states +
            sensor_states +
            ['elasticagent', 'elasticsearch', 'logstash', 'redis', 'nginx']
        ),
        'so-idh': (
            ssl_states +
            ['idh']
        ),
        'so-import': (
            ssl_states +
            manager_states +
            sensor_states | reject('equalto', 'strelka') | reject('equalto', 'healthcheck') | list +
-            ['elasticsearch', 'elasticsearch.auth', 'kibana', 'kibana.secrets', 'strelka.manager']
+            ['elasticsearch', 'elasticsearch.auth', 'kibana', 'kibana.secrets', 'logstash.ssl', 'strelka.manager']
        ),
        'so-manager': (
            ssl_states +
            manager_states +
            ['salt.cloud', 'libvirt.packages', 'libvirt.ssh.users', 'strelka.manager'] +
            stig_states +
@@ -106,7 +95,6 @@
            elastic_stack_states
        ),
        'so-managerhype': (
            ssl_states +
            manager_states +
            ['salt.cloud', 'strelka.manager', 'hypervisor', 'libvirt'] +
            stig_states +
@@ -114,7 +102,6 @@
            elastic_stack_states
        ),
        'so-managersearch': (
            ssl_states +
            manager_states +
            ['salt.cloud', 'libvirt.packages', 'libvirt.ssh.users', 'strelka.manager'] +
            stig_states +
@@ -122,12 +109,10 @@
            elastic_stack_states
        ),
        'so-searchnode': (
            ssl_states +
            ['kafka.ca', 'kafka.ssl', 'elasticsearch', 'logstash', 'nginx'] +
            stig_states
        ),
        'so-standalone': (
            ssl_states +
            manager_states +
            ['salt.cloud', 'libvirt.packages', 'libvirt.ssh.users'] +
            sensor_states +
@@ -136,29 +121,24 @@
            elastic_stack_states
        ),
        'so-sensor': (
            ssl_states +
            sensor_states +
            ['nginx'] +
            stig_states
        ),
        'so-fleet': (
            ssl_states +
            stig_states +
            ['logstash', 'nginx', 'healthcheck', 'elasticfleet']
        ),
        'so-receiver': (
            ssl_states +
            kafka_states +
            stig_states +
            ['logstash', 'redis']
        ),
        'so-hypervisor': (
            ssl_states +
            stig_states +
            ['hypervisor', 'libvirt']
        ),
        'so-desktop': (
            ['ssl', 'docker_clean', 'telegraf'] +
            stig_states
        )
    } %}
@@ -1,10 +1,10 @@
 backup:
  locations:
    description: List of locations to back up to the destination.
-    helpLink: backup.html
+    helpLink: backup
    global: True
  destination:
    description: Directory to store the configuration backups in.
-    helpLink: backup.html
+    helpLink: backup
    global: True
@@ -1,10 +1,12 @@
 {% macro remove_comments(bpfmerged, app) %}
 {# remove comments from the bpf #}
 {% set app_list = [] %}
 {% for bpf in bpfmerged[app] %}
-{%   if bpf.strip().startswith('#') %}
+{%   if not bpf.strip().startswith('#') %}
-{%     do bpfmerged[app].pop(loop.index0) %}
+{%     do app_list.append(bpf) %}
 {%   endif %}
 {% endfor %}
 {% do bpfmerged.update({app: app_list}) %}
 {% endmacro %}
@@ -1,10 +1,15 @@
 {% from 'vars/globals.map.jinja' import GLOBALS %}
-{% if GLOBALS.pcap_engine == "TRANSITION" %}
+{% set PCAP_BPF_STATUS = 0  %}
-{%   set PCAPBPF = ["ip and host 255.255.255.1 and port 1"] %}
+
 {% else %}
 {%   import_yaml 'bpf/defaults.yaml' as BPFDEFAULTS %}
 {%   set BPFMERGED = salt['pillar.get']('bpf', BPFDEFAULTS.bpf, merge=True) %}
 {%   import 'bpf/macros.jinja' as MACROS %}
 {{   MACROS.remove_comments(BPFMERGED, 'pcap') }}
 {%   set PCAPBPF = BPFMERGED.pcap %}
 {% if PCAPBPF %}
   {% set PCAP_BPF_CALC = salt['cmd.script']('salt://common/tools/sbin/so-bpf-compile', GLOBALS.sensor.interface + ' ' + PCAPBPF|join(" "),cwd='/root') %}
   {% if PCAP_BPF_CALC['retcode'] == 0 %}
      {% set PCAP_BPF_STATUS = 1 %}
   {% endif %}
 {% endif %}
@@ -1,16 +1,16 @@
 bpf:
  pcap:
-    description: List of BPF filters to apply to Stenographer.
+    description: List of BPF filters to apply to the PCAP engine.
    multiline: True
    forcedType: "[]string"
-    helpLink: bpf.html
+    helpLink: bpf
  suricata:
-    description: List of BPF filters to apply to Suricata.
+    description: List of BPF filters to apply to Suricata. This will apply to alerts and, if enabled, to metadata and PCAP logs generated by Suricata.
    multiline: True
    forcedType: "[]string"
-    helpLink: bpf.html
+    helpLink: bpf
  zeek:
    description: List of BPF filters to apply to Zeek.
    multiline: True
    forcedType: "[]string"
-    helpLink: bpf.html
+    helpLink: bpf
@@ -1,7 +1,16 @@
 {% from 'vars/globals.map.jinja' import GLOBALS %}
 {% import_yaml 'bpf/defaults.yaml' as BPFDEFAULTS %}
 {% set BPFMERGED = salt['pillar.get']('bpf', BPFDEFAULTS.bpf, merge=True) %}
 {% set SURICATA_BPF_STATUS = 0 %}
 {% import 'bpf/macros.jinja' as MACROS %}
 {{ MACROS.remove_comments(BPFMERGED, 'suricata') }}
 {% set SURICATABPF = BPFMERGED.suricata %}
 {% if SURICATABPF %}
   {% set SURICATA_BPF_CALC = salt['cmd.script']('salt://common/tools/sbin/so-bpf-compile', GLOBALS.sensor.interface + ' ' + SURICATABPF|join(" "),cwd='/root') %}
   {% if SURICATA_BPF_CALC['retcode'] == 0 %}
      {% set SURICATA_BPF_STATUS = 1 %}
   {% endif %}
 {% endif %}
@@ -1,7 +1,16 @@
 {% from 'vars/globals.map.jinja' import GLOBALS %}
 {% import_yaml 'bpf/defaults.yaml' as BPFDEFAULTS %}
 {% set BPFMERGED = salt['pillar.get']('bpf', BPFDEFAULTS.bpf, merge=True) %}
 {% set ZEEK_BPF_STATUS = 0 %}
 {% import 'bpf/macros.jinja' as MACROS %}
 {{ MACROS.remove_comments(BPFMERGED, 'zeek') }}
 {% set ZEEKBPF = BPFMERGED.zeek %}
 {% if ZEEKBPF %}
   {% set ZEEK_BPF_CALC = salt['cmd.script']('salt://common/tools/sbin/so-bpf-compile', GLOBALS.sensor.interface + ' ' + ZEEKBPF|join(" "),cwd='/root') %}
   {% if ZEEK_BPF_CALC['retcode'] == 0 %}
      {% set ZEEK_BPF_STATUS = 1 %}
   {% endif %}
 {% endif %}
@@ -1,4 +0,0 @@
 pki_issued_certs:
  file.directory:
    - name: /etc/pki/issued_certs
    - makedirs: True
@@ -3,70 +3,10 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% if sls in allowed_states %}
 {% from 'vars/globals.map.jinja' import GLOBALS %}
 include:
-  - ca.dirs
+{% if GLOBALS.is_manager %}
-
+  - ca.server
 /etc/salt/minion.d/signing_policies.conf:
  file.managed:
    - source: salt://ca/files/signing_policies.conf
 pki_private_key:
  x509.private_key_managed:
    - name: /etc/pki/ca.key
    - keysize: 4096
    - passphrase:
    - backup: True
    {% if salt['file.file_exists']('/etc/pki/ca.key') -%}
    - prereq:
      - x509: /etc/pki/ca.crt
    {%- endif %}
 pki_public_ca_crt:
  x509.certificate_managed:
    - name: /etc/pki/ca.crt
    - signing_private_key: /etc/pki/ca.key
    - CN: {{ GLOBALS.manager }}
    - C: US
    - ST: Utah
    - L: Salt Lake City
    - basicConstraints: "critical CA:true"
    - keyUsage: "critical cRLSign, keyCertSign"
    - extendedkeyUsage: "serverAuth, clientAuth"
    - subjectKeyIdentifier: hash
    - authorityKeyIdentifier: keyid:always, issuer
    - days_valid: 3650
    - days_remaining: 0
    - backup: True
    - replace: False
    - require:
      - sls: ca.dirs
    - timeout: 30
    - retry:
        attempts: 5
        interval: 30
 mine_update_ca_crt:
  module.run:
    - mine.update: []
    - onchanges:
      - x509: pki_public_ca_crt
 cakeyperms:
  file.managed:
    - replace: False
    - name: /etc/pki/ca.key
    - mode: 640
    - group: 939
 {% else %}
 {{sls}}_state_not_allowed:
  test.fail_without_changes:
    - name: {{sls}}_state_not_allowed
 {% endif %}
  - ca.trustca
@@ -0,0 +1,3 @@
 {% set CA = {
    'server': pillar.ca.server
 }%}
@@ -1,7 +1,35 @@
-pki_private_key:
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
 # or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 {% set setup_running = salt['cmd.retcode']('pgrep -x so-setup') == 0 %}
 {% if setup_running%}
 include:
  - ssl.remove
 remove_pki_private_key:
  file.absent:
    - name: /etc/pki/ca.key
-pki_public_ca_crt:
+remove_pki_public_ca_crt:
  file.absent:
    - name: /etc/pki/ca.crt
 remove_trusttheca:
  file.absent:
    - name: /etc/pki/tls/certs/intca.crt
 remove_pki_public_ca_crt_symlink:
  file.absent:
    - name: /opt/so/saltstack/local/salt/ca/files/ca.crt
 {% else %}
 so-setup_not_running:
  test.show_notification:
    - text: "This state is reserved for usage during so-setup."
 {% endif %}
@@ -0,0 +1,63 @@
 # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
 # or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% if sls in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 pki_private_key:
  x509.private_key_managed:
    - name: /etc/pki/ca.key
    - keysize: 4096
    - passphrase:
    - backup: True
    {% if salt['file.file_exists']('/etc/pki/ca.key') -%}
    - prereq:
      - x509: /etc/pki/ca.crt
    {%- endif %}
 pki_public_ca_crt:
  x509.certificate_managed:
    - name: /etc/pki/ca.crt
    - signing_private_key: /etc/pki/ca.key
    - CN: {{ GLOBALS.manager }}
    - C: US
    - ST: Utah
    - L: Salt Lake City
    - basicConstraints: "critical CA:true"
    - keyUsage: "critical cRLSign, keyCertSign"
    - extendedkeyUsage: "serverAuth, clientAuth"
    - subjectKeyIdentifier: hash
    - authorityKeyIdentifier: keyid:always, issuer
    - days_valid: 3650
    - days_remaining: 7
    - backup: True
    - replace: False
    - timeout: 30
    - retry:
        attempts: 5
        interval: 30
 pki_public_ca_crt_symlink:
  file.symlink:
    - name: /opt/so/saltstack/local/salt/ca/files/ca.crt
    - target: /etc/pki/ca.crt
    - require:
      - x509: pki_public_ca_crt
 cakeyperms:
  file.managed:
    - replace: False
    - name: /etc/pki/ca.key
    - mode: 640
    - group: 939
 {% else %}
 {{sls}}_state_not_allowed:
  test.fail_without_changes:
    - name: {{sls}}_state_not_allowed
 {% endif %}
@@ -1,12 +1,15 @@
 #!/bin/bash
 # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
 # or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 # when the salt-minion signs the cert, a copy is stored here
 issued_certs_copypath:
  file.directory:
    - name: /etc/pki/issued_certs
    - makedirs: True
-
+signing_policy:
-. /usr/sbin/so-common
+  file.managed:
-
+    - name: /etc/salt/minion.d/signing_policies.conf
-/usr/sbin/so-restart idstools $1
+    - source: salt://ca/files/signing_policies.conf
@@ -3,11 +3,16 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 {% from 'idstools/map.jinja' import IDSTOOLSMERGED %}
 include:
-{% if IDSTOOLSMERGED.enabled %}
+  - docker
-  - idstools.enabled
+
-{% else %}
+# Trust the CA
-  - idstools.disabled
+trusttheca:
-{% endif %}
+  file.managed:
    - name: /etc/pki/tls/certs/intca.crt
    - source: salt://ca/files/ca.crt
    - watch_in:
      - service: docker_running
    - show_changes: False
    - makedirs: True
@@ -1,12 +0,0 @@
 {
  "registry-mirrors": [
    "https://:5000"
  ],
  "bip": "172.17.0.1/24",
  "default-address-pools": [
    {
      "base": "172.17.0.0/24",
      "size": 24
    }
  ]
 }
@@ -0,0 +1,21 @@
 # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
 # or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 {% set nsm_exists = salt['file.directory_exists']('/nsm') %}
 {% if nsm_exists %}
 {%   set nsm_total = salt['cmd.shell']('df -BG /nsm | tail -1 | awk \'{print $2}\'') %}
 nsm_total:
  grains.present:
    - name: nsm_total
    - value: {{ nsm_total }}
 {% else %}
 nsm_missing:
  test.succeed_without_changes:
    - name: /nsm does not exist, skipping grain assignment
 {% endif %}
@@ -4,6 +4,7 @@
 {% from 'vars/globals.map.jinja' import GLOBALS %}
 include:
  - common.grains
  - common.packages
 {% if GLOBALS.role in GLOBALS.manager_roles %}
  - manager.elasticsearch # needed for elastic_curl_config state
@@ -19,11 +20,6 @@ kernel.printk:
  sysctl.present:
    - value: "3 4 1 3"
 # Remove variables.txt from /tmp - This is temp
 rmvariablesfile:
  file.absent:
    - name: /tmp/variables.txt
 # Add socore Group
 socoregroup:
  group.present:
@@ -148,35 +144,13 @@ common_sbin_jinja:
      - so-import-pcap
 {% endif %}
 {% if GLOBALS.role == 'so-heavynode' %}
 remove_so-pcap-import_heavynode:
  file.absent:
    - name: /usr/sbin/so-pcap-import
 remove_so-import-pcap_heavynode:
  file.absent:
    - name: /usr/sbin/so-import-pcap
 {% endif %}
 {% if not GLOBALS.is_manager%}
 # prior to 2.4.50 these scripts were in common/tools/sbin on the manager because of soup and distributed to non managers
 # these two states remove the scripts from non manager nodes
 remove_soup:
  file.absent:
    - name: /usr/sbin/soup
 remove_so-firewall:
  file.absent:
    - name: /usr/sbin/so-firewall
 {% endif %}
 so-status_script:
  file.managed:
    - name: /usr/sbin/so-status
    - source: salt://common/tools/sbin/so-status
    - mode: 755
-{% if GLOBALS.role in GLOBALS.sensor_roles %}
+{% if GLOBALS.is_sensor %}
 # Add sensor cleanup
 so-sensor-clean:
  cron.present:
@@ -1,52 +1,5 @@
 # we cannot import GLOBALS from vars/globals.map.jinja in this state since it is called in setup.virt.init
 # since it is early in setup of a new VM, the pillars imported in GLOBALS are not yet defined
 {% if grains.os_family == 'Debian' %}
 commonpkgs:
  pkg.installed:
    - skip_suggestions: True
    - pkgs:
      - apache2-utils
      - wget
      - ntpdate
      - jq
      - curl
      - ca-certificates
      - software-properties-common
      - apt-transport-https
      - openssl
      - netcat-openbsd
      - sqlite3
      - libssl-dev
      - procps
      - python3-dateutil
      - python3-docker
      - python3-packaging
      - python3-lxml
      - git
      - rsync
      - vim
      - tar
      - unzip
      - bc
      {% if grains.oscodename != 'focal' %}
      - python3-rich
      {% endif %}
 {%     if grains.oscodename == 'focal' %}
 # since Ubuntu requires and internet connection we can use pip to install modules
 python3-pip:
  pkg.installed
 python-rich:
  pip.installed:
    - name: rich
    - target: /usr/local/lib/python3.8/dist-packages/
    - require:
      - pkg: python3-pip
 {%     endif %}
 {% endif %}
 {% if grains.os_family == 'RedHat' %}
 remove_mariadb:
  pkg.removed:
@@ -84,5 +37,3 @@ commonpkgs:
      - unzip
      - wget
      - yum-utils
 {% endif %}
@@ -3,8 +3,6 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 {% if '2.4' in salt['cp.get_file_str']('/etc/soversion') %}
 {%   import_yaml '/opt/so/saltstack/local/pillar/global/soc_global.sls' as SOC_GLOBAL %}
 {%   if SOC_GLOBAL.global.airgap %}
 {%     set UPDATE_DIR='/tmp/soagupdate/SecurityOnion' %}
@@ -13,14 +11,6 @@
 {%   endif %}
 {%   set SOVERSION = salt['file.read']('/etc/soversion').strip() %}
 remove_common_soup:
  file.absent:
    - name: /opt/so/saltstack/default/salt/common/tools/sbin/soup
 remove_common_so-firewall:
  file.absent:
    - name: /opt/so/saltstack/default/salt/common/tools/sbin/so-firewall
 # This section is used to put the scripts in place in the Salt file system
 # in case a state run tries to overwrite what we do in the next section.
 copy_so-common_common_tools_sbin:
@@ -120,23 +110,3 @@ copy_bootstrap-salt_sbin:
    - source: {{UPDATE_DIR}}/salt/salt/scripts/bootstrap-salt.sh
    - force: True
    - preserve: True
 {# this is added in 2.4.120 to remove salt repo files pointing to saltproject.io to accomodate the move to broadcom and new bootstrap-salt script #}
 {%   if salt['pkg.version_cmp'](SOVERSION, '2.4.120') == -1 %}
 {%     set saltrepofile = '/etc/yum.repos.d/salt.repo' %}
 {%     if grains.os_family == 'Debian' %}
 {%       set saltrepofile = '/etc/apt/sources.list.d/salt.list' %}
 {%     endif %}
 remove_saltproject_io_repo_manager:
  file.absent:
    - name: {{ saltrepofile }}
 {%   endif %}
 {% else %}
 fix_23_soup_sbin:
  cmd.run:
    - name: curl -s -f -o /usr/sbin/soup https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.3/main/salt/common/tools/sbin/soup
 fix_23_soup_salt:
  cmd.run:
    - name: curl -s -f -o /opt/so/saltstack/defalt/salt/common/tools/sbin/soup https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.3/main/salt/common/tools/sbin/soup
 {% endif %}
@@ -16,7 +16,7 @@
 if [ "$#" -lt 2 ]; then
  cat 1>&2 <<EOF
-$0 compiles a BPF expression to be passed to stenotype to apply a socket filter.
+$0 compiles a BPF expression to be passed to PCAP to apply a socket filter.
 Its first argument is the interface (link type is required) and all other arguments
 are passed to TCPDump.
@@ -29,9 +29,26 @@ fi
 interface="$1"
 shift
-tcpdump -i $interface -ddd $@ | tail -n+2 |
+
-while read line; do
+# Capture tcpdump output and exit code
 tcpdump_output=$(tcpdump -i "$interface" -ddd "$@" 2>&1)
 tcpdump_exit=$?
 if [ $tcpdump_exit -ne 0 ]; then
  echo "$tcpdump_output" >&2
  exit $tcpdump_exit
 fi
 # Process the output, skipping the first line
 echo "$tcpdump_output" | tail -n+2 | while read -r line; do
  cols=( $line )
-  printf "%04x%02x%02x%08x" ${cols[0]} ${cols[1]} ${cols[2]} ${cols[3]}
+  printf "%04x%02x%02x%08x" "${cols[0]}" "${cols[1]}" "${cols[2]}" "${cols[3]}"
 done
 # Check if the pipeline succeeded
 if [ "${PIPESTATUS[0]}" -ne 0 ]; then
  exit 1
 fi
 echo ""
 exit 0
@@ -10,7 +10,7 @@
 cat << EOF
 so-checkin will run a full salt highstate to apply all salt states. If a highstate is already running, this request will be queued and so it may pause for a few minutes before you see any more output. For more information about so-checkin and salt, please see:
-https://docs.securityonion.net/en/2.4/salt.html
+https://securityonion.net/docs/salt
 EOF
@@ -10,7 +10,7 @@
 # and since this same logic is required during installation, it's included in this file.
 DEFAULT_SALT_DIR=/opt/so/saltstack/default
-DOC_BASE_URL="https://docs.securityonion.net/en/2.4"
+DOC_BASE_URL="https://securityonion.net/docs"
 if [ -z $NOROOT ]; then
 	# Check for prerequisites
@@ -220,12 +220,22 @@ compare_es_versions() {
 }
 copy_new_files() {
  # Define files to exclude from deletion (relative to their respective base directories)
  local EXCLUDE_FILES=(
    "salt/hypervisor/soc_hypervisor.yaml"
  )
  # Build rsync exclude arguments
  local EXCLUDE_ARGS=()
  for file in "${EXCLUDE_FILES[@]}"; do
    EXCLUDE_ARGS+=(--exclude="$file")
  done
  # Copy new files over to the salt dir
  cd $UPDATE_DIR
-  rsync -a salt $DEFAULT_SALT_DIR/ --delete
+  rsync -a salt $DEFAULT_SALT_DIR/ --delete "${EXCLUDE_ARGS[@]}"
-  rsync -a pillar $DEFAULT_SALT_DIR/ --delete
+  rsync -a pillar $DEFAULT_SALT_DIR/ --delete "${EXCLUDE_ARGS[@]}"
  chown -R socore:socore $DEFAULT_SALT_DIR/
  chmod 755 $DEFAULT_SALT_DIR/pillar/firewall/addfirewall.sh
  cd /tmp
 }
@@ -323,8 +333,8 @@ get_elastic_agent_vars() {
 	if [ -f "$defaultsfile" ]; then
 		ELASTIC_AGENT_TARBALL_VERSION=$(egrep " +version: " $defaultsfile | awk -F: '{print $2}' | tr -d '[:space:]')
-		ELASTIC_AGENT_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz"
+		ELASTIC_AGENT_URL="https://repo.securityonion.net/file/so-repo/prod/3/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz"
-		ELASTIC_AGENT_MD5_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5"
+		ELASTIC_AGENT_MD5_URL="https://repo.securityonion.net/file/so-repo/prod/3/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5"
 		ELASTIC_AGENT_FILE="/nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz"
 		ELASTIC_AGENT_MD5="/nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5"
 		ELASTIC_AGENT_EXPANSION_DIR=/nsm/elastic-fleet/artifacts/beats/elastic-agent
@@ -339,21 +349,16 @@ get_random_value() {
 }
 gpg_rpm_import() {
-	if [[ $is_oracle ]]; then
+	if [[ "$WHATWOULDYOUSAYYAHDOHERE" == "setup" ]]; then
-	    if [[ "$WHATWOULDYOUSAYYAHDOHERE" == "setup" ]]; then
+		local RPMKEYSLOC="../salt/repo/client/files/$OS/keys"
-	        local RPMKEYSLOC="../salt/repo/client/files/$OS/keys"
+	else
-	    else
+		local RPMKEYSLOC="$UPDATE_DIR/salt/repo/client/files/$OS/keys"
 	        local RPMKEYSLOC="$UPDATE_DIR/salt/repo/client/files/$OS/keys"
 	    fi
 		RPMKEYS=('RPM-GPG-KEY-oracle' 'RPM-GPG-KEY-EPEL-9' 'SALT-PROJECT-GPG-PUBKEY-2023.pub' 'docker.pub' 'securityonion.pub')
 		for RPMKEY in "${RPMKEYS[@]}"; do
    	        rpm --import $RPMKEYSLOC/$RPMKEY
 	        echo "Imported $RPMKEY"
 	    done
 	elif [[ $is_rpm ]]; then
 		echo "Importing the security onion GPG key"
 		rpm --import ../salt/repo/client/files/oracle/keys/securityonion.pub
 	fi
 	RPMKEYS=('RPM-GPG-KEY-oracle' 'RPM-GPG-KEY-EPEL-9' 'SALT-PROJECT-GPG-PUBKEY-2023.pub' 'docker.pub' 'securityonion.pub')
 	for RPMKEY in "${RPMKEYS[@]}"; do
 		rpm --import $RPMKEYSLOC/$RPMKEY
 		echo "Imported $RPMKEY"
 	done
 }
 header() {
@@ -385,7 +390,7 @@ is_manager_node() {
 }
 is_sensor_node() {
-	# Check to see if this is a sensor (forward) node
+	# Check to see if this is a sensor node
 	is_single_node_grid && return 0
 	grep "role: so-" /etc/salt/grains | grep -E "sensor|heavynode" &> /dev/null
 }
@@ -394,6 +399,25 @@ is_single_node_grid() {
 	grep "role: so-" /etc/salt/grains | grep -E "eval|standalone|import" &> /dev/null
 }
 initialize_elasticsearch_indices() {
    local index_names=$1
    local default_entry=${2:-'{"@timestamp":"0"}'}
    for idx in $index_names; do
        if ! so-elasticsearch-query "$idx" --fail --retry 3 --retry-delay 30 >/dev/null 2>&1; then
            echo "Index does not already exist. Initializing $idx index."
            if retry 3 10 "so-elasticsearch-query "$idx/_doc" -d '$default_entry' -XPOST --fail 2>/dev/null" '"successful":1'; then
                echo "Successfully initialized $idx index."
            else
                echo "Failed to initialize $idx index after 3 attempts."
            fi
        else
            echo "Index $idx already exists. No action needed."
        fi
    done
 }
 lookup_bond_interfaces() {
 	cat /proc/net/bonding/bond0 | grep "Slave Interface:" | sed -e "s/Slave Interface: //g"
 }
@@ -441,8 +465,7 @@ lookup_grain() {
 lookup_role() {
 	id=$(lookup_grain id)
-	pieces=($(echo $id | tr '_' ' '))
+	echo "${id##*_}"
 	echo ${pieces[1]}
 }
 is_feature_enabled() {
@@ -522,6 +545,22 @@ retry() {
        return $exitcode
 }
 rollover_index() {
  idx=$1
  exists=$(so-elasticsearch-query $idx -o /dev/null -w "%{http_code}")
  if [[ $exists -eq 200 ]]; then
    rollover=$(so-elasticsearch-query $idx/_rollover -o /dev/null -w "%{http_code}" -XPOST)
    if [[ $rollover -eq 200 ]]; then
      echo "Successfully triggered rollover for $idx..."
    else
      echo "Could not trigger rollover for $idx..."
    fi
  else
    echo "Could not find index $idx..."
  fi
 }
 run_check_net_err() {
 	local cmd=$1
 	local err_msg=${2:-"Unknown error occured, please check /root/$WHATWOULDYOUSAYYAHDOHERE.log for details."} # Really need to rename that variable
@@ -545,21 +584,39 @@ run_check_net_err() {
 }
 wait_for_salt_minion() {
-	local minion="$1"
+    local minion="$1"
-	local timeout="${2:-5}"
+    local max_wait="${2:-30}"
-	local logfile="${3:-'/dev/stdout'}"
+    local interval="${3:-2}"
-	retry 60 5 "journalctl -u salt-minion.service | grep 'Minion is ready to receive requests'" >> "$logfile" 2>&1 || fail
+    local logfile="${4:-'/dev/stdout'}"
-	local attempt=0
+    local elapsed=0
-	# each attempts would take about 15 seconds
+
-	local maxAttempts=20
+	echo "$(date '+%a %d %b %Y %H:%M:%S.%6N') - Waiting for salt-minion '$minion' to be ready..."
-	until check_salt_minion_status "$minion" "$timeout" "$logfile"; do
+
-		attempt=$((attempt+1))
+    while [ $elapsed -lt $max_wait ]; do
-		if [[ $attempt -eq $maxAttempts ]]; then
+        # Check if service is running
-			return 1
+		echo "$(date '+%a %d %b %Y %H:%M:%S.%6N') - Check if salt-minion service is running"
-		fi
+        if ! systemctl is-active --quiet salt-minion; then
-		sleep 10
+            echo "$(date '+%a %d %b %Y %H:%M:%S.%6N') - salt-minion service not running (elapsed: ${elapsed}s)"
-	done
+            sleep $interval
-	return 0
+            elapsed=$((elapsed + interval))
            continue
        fi
 		echo "$(date '+%a %d %b %Y %H:%M:%S.%6N') - salt-minion service is running"
        # Check if minion responds to ping
 		echo "$(date '+%a %d %b %Y %H:%M:%S.%6N') - Check if $minion responds to ping"
        if salt "$minion" test.ping --timeout=3 --out=json 2>> "$logfile" | grep -q "true"; then
            echo "$(date '+%a %d %b %Y %H:%M:%S.%6N') - salt-minion '$minion' is connected and ready!"
            return 0
        fi
        echo "$(date '+%a %d %b %Y %H:%M:%S.%6N') - Waiting... (${elapsed}s / ${max_wait}s)"
        sleep $interval
        elapsed=$((elapsed + interval))
    done
    echo "$(date '+%a %d %b %Y %H:%M:%S.%6N') - ERROR: salt-minion '$minion' not ready after $max_wait seconds"
    return 1
 }
 salt_minion_count() {
@@ -569,69 +626,19 @@ salt_minion_count() {
 }
 set_os() {
-	if [ -f /etc/redhat-release ]; then
+	if [ -f /etc/redhat-release ] && grep -q "Red Hat Enterprise Linux release 9" /etc/redhat-release && [ -f /etc/oracle-release ]; then
-		if grep -q "Rocky Linux release 9" /etc/redhat-release; then
+		OS=oracle
-			OS=rocky
+		OSVER=9
-			OSVER=9
+		is_oracle=true
-			is_rocky=true
+		is_rpm=true
 			is_rpm=true
 		elif grep -q "CentOS Stream release 9" /etc/redhat-release; then
 			OS=centos
 			OSVER=9
 			is_centos=true
 			is_rpm=true
 		elif grep -q "AlmaLinux release 9" /etc/redhat-release; then
 			OS=alma
 			OSVER=9
 			is_alma=true
 			is_rpm=true
 		elif grep -q "Red Hat Enterprise Linux release 9" /etc/redhat-release; then
 			if [ -f /etc/oracle-release ]; then
 				OS=oracle
 				OSVER=9
 				is_oracle=true
 				is_rpm=true
 			else
 				OS=rhel
 				OSVER=9
 				is_rhel=true
 				is_rpm=true
 			fi
 		fi
 		cron_service_name="crond"
 	elif [ -f /etc/os-release ]; then
 		if grep -q "UBUNTU_CODENAME=focal" /etc/os-release; then
 			OSVER=focal
 			UBVER=20.04
 			OS=ubuntu
 			is_ubuntu=true
 			is_deb=true
 		elif grep -q "UBUNTU_CODENAME=jammy" /etc/os-release; then
 			OSVER=jammy
 			UBVER=22.04
 			OS=ubuntu
 			is_ubuntu=true
 			is_deb=true
 		elif grep -q "VERSION_CODENAME=bookworm" /etc/os-release; then
 			OSVER=bookworm
 			DEBVER=12
 			is_debian=true
 			OS=debian
 			is_deb=true
 		fi
 		cron_service_name="cron"
 	fi
 	cron_service_name="crond"
 }
 set_minionid() {
 	MINIONID=$(lookup_grain id)
 }
 set_palette() {
 	if [[ $is_deb ]]; then
 	    update-alternatives --set newt-palette /etc/newt/palette.original
    fi
 }
 set_version() {
 	CURRENTVERSION=0.0.0
@@ -25,7 +25,6 @@ container_list() {
  if [ $MANAGERCHECK == 'so-import' ]; then
    TRUSTED_CONTAINERS=(
      "so-elasticsearch"
      "so-idstools"
      "so-influxdb"
      "so-kibana"
      "so-kratos"
@@ -33,7 +32,6 @@ container_list() {
      "so-nginx"
      "so-pcaptools"
      "so-soc"
      "so-steno"
      "so-suricata"
      "so-telegraf"
      "so-zeek"
@@ -49,7 +47,6 @@ container_list() {
      "so-elastic-fleet-package-registry"
      "so-elasticsearch"
      "so-idh"
      "so-idstools"
      "so-influxdb"
      "so-kafka"
      "so-kibana"
@@ -60,10 +57,7 @@ container_list() {
      "so-pcaptools"
      "so-redis"
      "so-soc"
      "so-steno"
      "so-strelka-backend"
      "so-strelka-filestream"
      "so-strelka-frontend"
      "so-strelka-manager"
      "so-suricata"
      "so-telegraf"
@@ -71,12 +65,10 @@ container_list() {
    )
  else
    TRUSTED_CONTAINERS=(
      "so-idstools"
      "so-elasticsearch"
      "so-logstash"
      "so-nginx"
      "so-redis"
      "so-steno"
      "so-suricata"
      "so-soc"
      "so-telegraf"
@@ -129,6 +129,9 @@ if [[ $EXCLUDE_STARTUP_ERRORS == 'Y' ]]; then
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|responded with status-code 503" # telegraf getting 503 from ES during startup
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|process_cluster_event_timeout_exception" # logstash waiting for elasticsearch to start
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|not configured for GeoIP"     # SO does not bundle the maxminddb with Zeek
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|HTTP 404: Not Found"          # Salt loops until Kratos returns 200, during startup Kratos may not be ready
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Cancelling deferred write event maybeFenceReplicas because the event queue is now closed" # Kafka controller log during shutdown/restart
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Redis may have been restarted" # Redis likely restarted by salt
 fi
 if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then
@@ -159,7 +162,9 @@ if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|adding ingest pipeline"       # false positive (elasticsearch ingest pipeline names contain 'error') 
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|updating index template"      # false positive (elasticsearch index or template names contain 'error') 
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|updating component template"  # false positive (elasticsearch index or template names contain 'error') 
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|upgrading component template"  # false positive (elasticsearch index or template names contain 'error')
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|upgrading composable template" # false positive (elasticsearch composable template names contain 'error')
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Error while parsing document for index \[.ds-logs-kratos-so-.*object mapping for \[file\]" # false positive (mapping error occuring BEFORE kratos index has rolled over in 2.4.210)
 fi
 if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
@@ -175,7 +180,6 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|salt-minion-check"            # bug in early 2.4 place Jinja script in non-jinja salt dir causing cron output errors
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|monitoring.metrics"           # known issue with elastic agent casting the field incorrectly if an integer value shows up before a float
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|repodownload.conf"            # known issue with reposync on pre-2.4.20 
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|missing versions record"      # stenographer corrupt index
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|soc.field."                   # known ingest type collisions issue with earlier versions of SO
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|error parsing signature"      # Malformed Suricata rule, from upstream provider
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|sticky buffer has no matches" # Non-critical Suricata error
@@ -222,6 +226,9 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Initialized license manager"  # SOC log: before fields.status was changed to fields.licenseStatus
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|from NIC checksum offloading" # zeek reporter.log
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|marked for removal"           # docker container getting recycled
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|tcp 127.0.0.1:6791: bind: address already in use" # so-elastic-fleet agent restarting. Seen starting w/ 8.18.8 https://github.com/elastic/kibana/issues/201459
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|TransformTask\] \[logs-(tychon|aws_billing|microsoft_defender_endpoint).*user so_kibana lacks the required permissions \[logs-\1" # Known issue with 3 integrations using kibana_system role vs creating unique api creds with proper permissions.
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|manifest unknown"             # appears in so-dockerregistry log for so-tcpreplay following docker upgrade to 29.2.1-1
 fi
 RESULT=0
@@ -268,6 +275,13 @@ for log_file in $(cat /tmp/log_check_files); do
    tail -n $RECENT_LOG_LINES $log_file > /tmp/log_check
    check_for_errors
 done
 # Look for OOM specific errors in /var/log/messages which can lead to odd behavior / test failures
 if [[ -f /var/log/messages ]]; then
    status "Checking log file /var/log/messages"
    if journalctl --since "24 hours ago" | grep -iE 'out of memory|oom-kill'; then
        RESULT=1
    fi
 fi
 # Cleanup temp files
 rm -f /tmp/log_check_files
@@ -55,19 +55,22 @@ if [ $SKIP -ne 1 ]; then
 fi
 delete_pcap() {
-  PCAP_DATA="/nsm/pcap/"
+  PCAP_DATA="/nsm/suripcap/"
-  [ -d $PCAP_DATA ] && so-pcap-stop && rm -rf $PCAP_DATA/* && so-pcap-start
+  [ -d $PCAP_DATA ] && rm -rf $PCAP_DATA/*
 }
 delete_suricata() {
  SURI_LOG="/nsm/suricata/"
-  [ -d $SURI_LOG ] && so-suricata-stop && rm -rf $SURI_LOG/* && so-suricata-start
+  [ -d $SURI_LOG ] && rm -rf $SURI_LOG/*
 }
 delete_zeek() {
  ZEEK_LOG="/nsm/zeek/logs/"
  [ -d $ZEEK_LOG ] && so-zeek-stop && rm -rf $ZEEK_LOG/* && so-zeek-start
 }
 so-suricata-stop
 delete_pcap
 delete_suricata
 delete_zeek
 so-suricata-start
@@ -23,7 +23,6 @@ if [ $# -ge 1 ]; then
        fi
        case $1 in
                "steno") docker stop so-steno && docker rm so-steno && salt-call state.apply pcap queue=True;;
                "elastic-fleet") docker stop so-elastic-fleet && docker rm so-elastic-fleet && salt-call state.apply elasticfleet queue=True;;
                *)  docker stop so-$1 ; docker rm so-$1 ; salt-call state.apply $1 queue=True;;
        esac
@@ -72,7 +72,7 @@ clean() {
 		done
 	fi
-	## Clean up extracted pcaps from Steno
+	## Clean up extracted pcaps
 	PCAPS='/nsm/pcapout'
 	OLDEST_PCAP=$(find $PCAPS -type f -printf '%T+ %p\n' | sort -n | head -n 1)
 	if [ -z "$OLDEST_PCAP" -o "$OLDEST_PCAP" == ".." -o "$OLDEST_PCAP" == "." ]; then
@@ -23,7 +23,6 @@ if [ $# -ge 1 ]; then
 	case $1 in
   		"all") salt-call state.highstate queue=True;;
   		"steno") if docker ps | grep -q so-$1; then printf "\n$1 is already running!\n\n"; else docker rm so-$1 >/dev/null 2>&1 ; salt-call state.apply pcap queue=True; fi ;; 
   		"elastic-fleet") if docker ps | grep -q so-$1; then printf "\n$1 is already running!\n\n"; else docker rm so-$1 >/dev/null 2>&1 ; salt-call state.apply elasticfleet queue=True; fi ;; 
   		*) if docker ps | grep -E -q '^so-$1$'; then printf "\n$1 is already running\n\n"; else docker rm so-$1 >/dev/null 2>&1 ; salt-call state.apply $1 queue=True; fi ;; 
 	esac
@@ -6,7 +6,7 @@
 # Elastic License 2.0.
 source /usr/sbin/so-common
-doc_desktop_url="$DOC_BASE_URL/desktop.html"
+doc_desktop_url="$DOC_BASE_URL/desktop"
 {# we only want the script to install the desktop if it is OEL -#}
 {% if grains.os == 'OEL' -%}
@@ -85,7 +85,7 @@ function suricata() {
  docker run --rm \
    -v /opt/so/conf/suricata/suricata.yaml:/etc/suricata/suricata.yaml:ro \
    -v /opt/so/conf/suricata/threshold.conf:/etc/suricata/threshold.conf:ro \
-    -v /opt/so/conf/suricata/rules:/etc/suricata/rules:ro \
+    -v /opt/so/rules/suricata/:/etc/suricata/rules:ro \
    -v ${LOG_PATH}:/var/log/suricata/:rw \
    -v ${NSM_PATH}/:/nsm/:rw \
    -v "$PCAP:/input.pcap:ro" \
@@ -173,7 +173,7 @@ for PCAP in $INPUT_FILES; do
  status "- assigning unique identifier to import: $HASH"
  pcap_data=$(pcapinfo "${PCAP}")
-  if ! echo "$pcap_data" | grep -q "First packet time:" || echo "$pcap_data" |egrep -q "Last packet time:    1970-01-01|Last packet time:    n/a"; then
+  if ! echo "$pcap_data" | grep -q "Earliest packet time:" || echo "$pcap_data" |egrep -q "Latest packet time:   1970-01-01|Latest packet time:   n/a"; then
    status "- this PCAP file is invalid; skipping"
    INVALID_PCAPS_COUNT=$((INVALID_PCAPS_COUNT + 1))
  else
@@ -205,8 +205,8 @@ for PCAP in $INPUT_FILES; do
      HASHES="${HASHES} ${HASH}"
    fi
-    START=$(pcapinfo "${PCAP}" -a |grep "First packet time:" | awk '{print $4}')
+    START=$(pcapinfo "${PCAP}" -a |grep "Earliest packet time:" | awk '{print $4}')
-    END=$(pcapinfo "${PCAP}" -e |grep "Last packet time:" | awk '{print $4}')
+    END=$(pcapinfo "${PCAP}" -e |grep "Latest packet time:" | awk '{print $4}')
    status "- found PCAP data spanning dates $START through $END"
    # compare $START to $START_OLDEST
@@ -1,34 +0,0 @@
 # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
 # or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 so-curator:
  docker_container.absent:
    - force: True
 so-curator_so-status.disabled:
  file.line:
    - name: /opt/so/conf/so-status/so-status.conf
    - match: ^so-curator$
    - mode: delete
 so-curator-cluster-close:
  cron.absent:
    - identifier: so-curator-cluster-close
 so-curator-cluster-delete:
  cron.absent:
    - identifier: so-curator-cluster-delete
 delete_curator_configuration:
  file.absent:
    - name: /opt/so/conf/curator
    - recurse: True
 {% set files = salt.file.find(path='/usr/sbin', name='so-curator*') %}
 {% if files|length > 0 %}
 delete_curator_scripts:
  file.absent:
    - names: {{files|yaml}}
 {% endif %}
@@ -3,29 +3,16 @@
 {# we only want this state to run it is CentOS #}
 {% if GLOBALS.os == 'OEL' %}
  {% set global_ca_text = [] %}
  {% set global_ca_server = [] %}
  {% set manager = GLOBALS.manager %}
  {% set x509dict = salt['mine.get'](manager | lower~'*', 'x509.get_pem_entries') %}
    {% for host in x509dict %}
      {% if host.split('_')|last in ['manager', 'managersearch', 'standalone', 'import', 'eval'] %}
        {% do global_ca_text.append(x509dict[host].get('/etc/pki/ca.crt')|replace('\n', '')) %}
        {% do global_ca_server.append(host) %}
      {% endif %}
    {% endfor %}
  {% set trusttheca_text = global_ca_text[0] %}
  {% set ca_server = global_ca_server[0] %}
 trusted_ca:
-  x509.pem_managed:
+  file.managed:
    - name: /etc/pki/ca-trust/source/anchors/ca.crt
-    - text:  {{ trusttheca_text }}
+    - source: salt://ca/files/ca.crt
 update_ca_certs:
  cmd.run:
    - name: update-ca-trust
    - onchanges:
-      - x509: trusted_ca
+      - file: trusted_ca
 {% else %}
@@ -1,6 +1,10 @@
 docker:
  range: '172.17.1.0/24'
  gateway: '172.17.1.1'
  ulimits:
    - name: nofile
      soft: 1048576
      hard: 1048576
  containers:
    'so-dockerregistry':
      final_octet: 20
@@ -9,6 +13,7 @@ docker:
      custom_bind_mounts: []
      extra_hosts: []
      extra_env: []
      ulimits: []
    'so-elastic-fleet':
      final_octet: 21
      port_bindings:
@@ -16,6 +21,7 @@ docker:
      custom_bind_mounts: []
      extra_hosts: []
      extra_env: []
      ulimits: []
    'so-elasticsearch':
      final_octet: 22
      port_bindings:
@@ -24,11 +30,16 @@ docker:
      custom_bind_mounts: []
      extra_hosts: []
      extra_env: []
-    'so-idstools':
+      ulimits:
-      final_octet: 25
+        - name: memlock
-      custom_bind_mounts: []
+          soft: -1
-      extra_hosts: []
+          hard: -1
-      extra_env: []
+        - name: nofile
          soft: 65536
          hard: 65536
        - name: nproc
          soft: 4096
          hard: 4096
    'so-influxdb':
      final_octet: 26
      port_bindings:
@@ -36,6 +47,7 @@ docker:
      custom_bind_mounts: []
      extra_hosts: []
      extra_env: []
      ulimits: []
    'so-kibana':
      final_octet: 27
      port_bindings:
@@ -43,6 +55,7 @@ docker:
      custom_bind_mounts: []
      extra_hosts: []
      extra_env: []
      ulimits: []
    'so-kratos':
      final_octet: 28
      port_bindings:
@@ -51,6 +64,7 @@ docker:
      custom_bind_mounts: []
      extra_hosts: []
      extra_env: []
      ulimits: []
    'so-hydra':
      final_octet: 30
      port_bindings:
@@ -59,6 +73,7 @@ docker:
      custom_bind_mounts: []
      extra_hosts: []
      extra_env: []
      ulimits: []
    'so-logstash':
      final_octet: 29
      port_bindings:
@@ -75,6 +90,7 @@ docker:
      custom_bind_mounts: []
      extra_hosts: []
      extra_env: []
      ulimits: []
    'so-nginx':
      final_octet: 31
      port_bindings:
@@ -86,6 +102,7 @@ docker:
      custom_bind_mounts: []
      extra_hosts: []
      extra_env: []
      ulimits: []
    'so-nginx-fleet-node':
      final_octet: 31
      port_bindings:
@@ -93,6 +110,7 @@ docker:
      custom_bind_mounts: []
      extra_hosts: []
      extra_env: []
      ulimits: []
    'so-redis':
      final_octet: 33
      port_bindings:
@@ -101,11 +119,13 @@ docker:
      custom_bind_mounts: []
      extra_hosts: []
      extra_env: []
      ulimits: []
    'so-sensoroni':
      final_octet: 99
      custom_bind_mounts: []
      extra_hosts: []
      extra_env: []
      ulimits: []
    'so-soc':
      final_octet: 34
      port_bindings:
@@ -113,16 +133,19 @@ docker:
      custom_bind_mounts: []
      extra_hosts: []
      extra_env: []
      ulimits: []
    'so-strelka-backend':
      final_octet: 36
      custom_bind_mounts: []
      extra_hosts: []
      extra_env: []
      ulimits: []
    'so-strelka-filestream':
      final_octet: 37
      custom_bind_mounts: []
      extra_hosts: []
      extra_env: []
      ulimits: []
    'so-strelka-frontend':
      final_octet: 38
      port_bindings:
@@ -130,11 +153,13 @@ docker:
      custom_bind_mounts: []
      extra_hosts: []
      extra_env: []
      ulimits: []
    'so-strelka-manager':
      final_octet: 39
      custom_bind_mounts: []
      extra_hosts: []
      extra_env: []
      ulimits: []
    'so-strelka-gatekeeper':
      final_octet: 40
      port_bindings:
@@ -142,6 +167,7 @@ docker:
      custom_bind_mounts: []
      extra_hosts: []
      extra_env: []
      ulimits: []
    'so-strelka-coordinator':
      final_octet: 41
      port_bindings:
@@ -149,11 +175,13 @@ docker:
      custom_bind_mounts: []
      extra_hosts: []
      extra_env: []
      ulimits: []
    'so-elastalert':
      final_octet: 42
      custom_bind_mounts: []
      extra_hosts: []
      extra_env: []
      ulimits: []
    'so-elastic-fleet-package-registry':
      final_octet: 44
      port_bindings:
@@ -161,11 +189,13 @@ docker:
      custom_bind_mounts: []
      extra_hosts: []
      extra_env: []
      ulimits: []
    'so-idh':
      final_octet: 45
      custom_bind_mounts: []
      extra_hosts: []
      extra_env: []
      ulimits: []
    'so-elastic-agent':
      final_octet: 46
      port_bindings:
@@ -174,28 +204,28 @@ docker:
      custom_bind_mounts: []
      extra_hosts: []
      extra_env: []
      ulimits: []
    'so-telegraf':
      final_octet: 99
      custom_bind_mounts: []
      extra_hosts: []
      extra_env: []
-    'so-steno':
+      ulimits: []
      final_octet: 99
      custom_bind_mounts: []
      extra_hosts: []
      extra_env: []
    'so-suricata':
      final_octet: 99
      custom_bind_mounts: []
      extra_hosts: []
      extra_env: []
-      ulimits:
+      ulimits: []
        - memlock=524288000
    'so-zeek':
      final_octet: 99
      custom_bind_mounts: []
      extra_hosts: []
      extra_env: []
      ulimits:
        - name: core
          soft: 0
          hard: 0
    'so-kafka':
      final_octet: 88
      port_bindings:
@@ -206,3 +236,4 @@ docker:
      custom_bind_mounts: []
      extra_hosts: []
      extra_env: []
      ulimits: []
@@ -1,8 +1,8 @@
 {% import_yaml 'docker/defaults.yaml' as DOCKERDEFAULTS %}
-{% set DOCKER = salt['pillar.get']('docker', DOCKERDEFAULTS.docker, merge=True) %}
+{% set DOCKERMERGED = salt['pillar.get']('docker', DOCKERDEFAULTS.docker, merge=True) %}
-{% set RANGESPLIT = DOCKER.range.split('.') %}
+{% set RANGESPLIT = DOCKERMERGED.range.split('.') %}
 {% set FIRSTTHREE = RANGESPLIT[0] ~ '.' ~ RANGESPLIT[1] ~ '.' ~ RANGESPLIT[2] ~ '.' %}
-{% for container, vals in DOCKER.containers.items() %}
+{% for container, vals in DOCKERMERGED.containers.items() %}
-{%   do DOCKER.containers[container].update({'ip': FIRSTTHREE ~ DOCKER.containers[container].final_octet}) %}
+{%   do DOCKERMERGED.containers[container].update({'ip': FIRSTTHREE ~ DOCKERMERGED.containers[container].final_octet}) %}
 {% endfor %}
@@ -0,0 +1,24 @@
 {% from 'docker/docker.map.jinja' import DOCKERMERGED -%}
 {
  "registry-mirrors": [
    "https://:5000"
  ],
  "bip": "172.17.0.1/24",
  "default-address-pools": [
    {
      "base": "172.17.0.0/24",
      "size": 24
    }
  ]
 {%- if DOCKERMERGED.ulimits %},
  "default-ulimits": {
 {%-   for ULIMIT in DOCKERMERGED.ulimits %}
      "{{ ULIMIT.name }}": {
        "Name": "{{ ULIMIT.name }}",
        "Soft": {{ ULIMIT.soft }},
        "Hard": {{ ULIMIT.hard }}
      }{{ "," if not loop.last else "" }}
 {%-   endfor %}
  }
 {%- endif %}
 }
@@ -3,61 +3,27 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
-{% from 'docker/docker.map.jinja' import DOCKER %}
+{% from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {% from 'vars/globals.map.jinja' import GLOBALS %}
-# include ssl since docker service requires the intca
+# docker service requires the ca.crt
 include:
-  - ssl
+  - ca
 dockergroup:
  group.present:
    - name: docker
    - gid: 920
 {% if GLOBALS.os_family == 'Debian' %}
 {%    if grains.oscodename == 'bookworm' %}
 dockerheldpackages:
  pkg.installed:
    - pkgs:
-      - containerd.io: 1.7.21-1
+      - containerd.io: 2.2.1-1.el9
-      - docker-ce: 5:27.2.0-1~debian.12~bookworm
+      - docker-ce: 3:29.2.1-1.el9
-      - docker-ce-cli: 5:27.2.0-1~debian.12~bookworm
+      - docker-ce-cli: 1:29.2.1-1.el9
-      - docker-ce-rootless-extras: 5:27.2.0-1~debian.12~bookworm
+      - docker-ce-rootless-extras: 29.2.1-1.el9
    - hold: True
    - update_holds: True
 {%    elif grains.oscodename == 'jammy' %}
 dockerheldpackages:
  pkg.installed:
    - pkgs:
      - containerd.io: 1.7.21-1
      - docker-ce: 5:27.2.0-1~ubuntu.22.04~jammy
      - docker-ce-cli: 5:27.2.0-1~ubuntu.22.04~jammy
      - docker-ce-rootless-extras: 5:27.2.0-1~ubuntu.22.04~jammy
    - hold: True
    - update_holds: True
 {%    else %}
 dockerheldpackages:
  pkg.installed:
    - pkgs:
      - containerd.io: 1.7.21-1
      - docker-ce: 5:27.2.0-1~ubuntu.20.04~focal
      - docker-ce-cli: 5:27.2.0-1~ubuntu.20.04~focal
      - docker-ce-rootless-extras: 5:27.2.0-1~ubuntu.20.04~focal
    - hold: True
    - update_holds: True
 {%   endif %}
 {% else %}
 dockerheldpackages:
  pkg.installed:
    - pkgs:
      - containerd.io: 1.7.21-3.1.el9
      - docker-ce: 3:27.2.0-1.el9
      - docker-ce-cli: 1:27.2.0-1.el9
      - docker-ce-rootless-extras: 27.2.0-1.el9
    - hold: True
    - update_holds: True
 {% endif %}
 #disable docker from managing iptables
 iptables_disabled:
@@ -75,10 +41,9 @@ dockeretc:
  file.directory:
    - name: /etc/docker
 # Manager daemon.json
 docker_daemon:
  file.managed:
-    - source: salt://common/files/daemon.json
+    - source: salt://docker/files/daemon.json.jinja
    - name: /etc/docker/daemon.json
    - template: jinja 
@@ -89,10 +54,9 @@ docker_running:
    - enable: True
    - watch:
      - file: docker_daemon
      - x509: trusttheca
    - require:
      - file: docker_daemon
-      - x509: trusttheca
+      - file: trusttheca
 # Reserve OS ports for Docker proxy in case boot settings are not already applied/present
@@ -110,12 +74,12 @@ dockerreserveports:
 sos_docker_net:
  docker_network.present:
    - name: sobridge
-    - subnet: {{ DOCKER.range }}
+    - subnet: {{ DOCKERMERGED.range }}
-    - gateway: {{ DOCKER.gateway }}
+    - gateway: {{ DOCKERMERGED.gateway }}
    - options:
        com.docker.network.bridge.name: 'sobridge'
        com.docker.network.driver.mtu: '1500'
        com.docker.network.bridge.enable_ip_masquerade: 'true'
        com.docker.network.bridge.enable_icc: 'true'
        com.docker.network.bridge.host_binding_ipv4: '0.0.0.0'
-    - unless: 'docker network ls | grep sobridge'
+    - unless: ip l | grep sobridge
@@ -1,47 +1,84 @@
 docker:
  gateway:
    description: Gateway for the default docker interface.
-    helpLink: docker.html
+    helpLink: docker
    advanced: True
  range:
    description: Default docker IP range for containers.
-    helpLink: docker.html
+    helpLink: docker
    advanced: True
  ulimits:
    description: |
      Default ulimit settings applied to all containers via the Docker daemon. Each entry specifies a resource name (e.g. nofile, memlock, core, nproc) with soft and hard limits. Individual container ulimits override these defaults. Valid resource names include: cpu, fsize, data, stack, core, rss, nproc, nofile, memlock, as, locks, sigpending, msgqueue, nice, rtprio, rttime.
    forcedType: "[]{}"
    syntax: json
    advanced: True
    helpLink: docker.html
    uiElements:
    - field: name
      label: Resource Name
      required: True
      regex: ^(cpu|fsize|data|stack|core|rss|nproc|nofile|memlock|as|locks|sigpending|msgqueue|nice|rtprio|rttime)$
      regexFailureMessage: You must enter a valid ulimit name (cpu, fsize, data, stack, core, rss, nproc, nofile, memlock, as, locks, sigpending, msgqueue, nice, rtprio, rttime).
    - field: soft
      label: Soft Limit
      forcedType: int
    - field: hard
      label: Hard Limit
      forcedType: int
  containers:
    so-dockerregistry: &dockerOptions
      final_octet:
        description: Last octet of the container IP address.
-        helpLink: docker.html
+        helpLink: docker
        readonly: True
        advanced: True
        global: True
      port_bindings:
        description: List of port bindings for the container.
-        helpLink: docker.html
+        helpLink: docker
        advanced: True
        multiline: True
        forcedType: "[]string"
      custom_bind_mounts:
        description: List of custom local volume bindings.
        advanced: True
-        helpLink: docker.html
+        helpLink: docker
        multiline: True
        forcedType: "[]string"
      extra_hosts:
        description: List of additional host entries for the container.
        advanced: True
-        helpLink: docker.html
+        helpLink: docker
        multiline: True
        forcedType: "[]string"
      extra_env:
        description: List of additional ENV entries for the container.
        advanced: True
-        helpLink: docker.html
+        helpLink: docker
        multiline: True
        forcedType: "[]string"
      ulimits:
        description: |
          Ulimit settings for the container. Each entry specifies a resource name (e.g. nofile, memlock, core, nproc) with optional soft and hard limits. Valid resource names include: cpu, fsize, data, stack, core, rss, nproc, nofile, memlock, as, locks, sigpending, msgqueue, nice, rtprio, rttime.
        advanced: True
        helpLink: docker.html
        forcedType: "[]{}"
        syntax: json
        uiElements:
        - field: name
          label: Resource Name
          required: True
          regex: ^(cpu|fsize|data|stack|core|rss|nproc|nofile|memlock|as|locks|sigpending|msgqueue|nice|rtprio|rttime)$
          regexFailureMessage: You must enter a valid ulimit name (cpu, fsize, data, stack, core, rss, nproc, nofile, memlock, as, locks, sigpending, msgqueue, nice, rtprio, rttime).
        - field: soft
          label: Soft Limit
          forcedType: int
        - field: hard
          label: Hard Limit
          forcedType: int
    so-elastic-fleet: *dockerOptions
    so-elasticsearch: *dockerOptions
    so-idstools: *dockerOptions
    so-influxdb: *dockerOptions
    so-kibana: *dockerOptions
    so-kratos: *dockerOptions
@@ -63,43 +100,6 @@ docker:
    so-idh: *dockerOptions
    so-elastic-agent: *dockerOptions
    so-telegraf: *dockerOptions
-    so-steno: *dockerOptions
+    so-suricata: *dockerOptions
    so-suricata:
      final_octet:
        description: Last octet of the container IP address.
        helpLink: docker.html
        readonly: True
        advanced: True
        global: True
      port_bindings:
        description: List of port bindings for the container.
        helpLink: docker.html
        advanced: True
        multiline: True
        forcedType: "[]string"
      custom_bind_mounts:
        description: List of custom local volume bindings.
        advanced: True
        helpLink: docker.html
        multiline: True
        forcedType: "[]string"
      extra_hosts:
        description: List of additional host entries for the container.
        advanced: True
        helpLink: docker.html
        multiline: True
        forcedType: "[]string"
      extra_env:
        description: List of additional ENV entries for the container.
        advanced: True
        helpLink: docker.html
        multiline: True
        forcedType: "[]string"
      ulimits:
        description: Ulimits for the container, in bytes.
        advanced: True
        helpLink: docker.html
        multiline: True
        forcedType: "[]string"
    so-zeek: *dockerOptions
-    so-kafka: *dockerOptions
+    so-kafka: *dockerOptions
@@ -6,7 +6,7 @@
 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
-{%   from 'docker/docker.map.jinja' import DOCKER %}
+{%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 include:
  - elastalert.config
@@ -24,7 +24,7 @@ so-elastalert:
    - user: so-elastalert
    - networks:
      - sobridge:
-        - ipv4_address: {{ DOCKER.containers['so-elastalert'].ip }}
+        - ipv4_address: {{ DOCKERMERGED.containers['so-elastalert'].ip }}
    - detach: True
    - binds:
      - /opt/so/rules/elastalert:/opt/elastalert/rules/:ro
@@ -33,24 +33,30 @@ so-elastalert:
      - /opt/so/conf/elastalert/predefined/:/opt/elastalert/predefined/:ro
      - /opt/so/conf/elastalert/custom/:/opt/elastalert/custom/:ro
      - /opt/so/conf/elastalert/elastalert_config.yaml:/opt/elastalert/config.yaml:ro
-      {% if DOCKER.containers['so-elastalert'].custom_bind_mounts %}
+      {% if DOCKERMERGED.containers['so-elastalert'].custom_bind_mounts %}
-        {% for BIND in DOCKER.containers['so-elastalert'].custom_bind_mounts %}
+        {% for BIND in DOCKERMERGED.containers['so-elastalert'].custom_bind_mounts %}
      - {{ BIND }}
        {% endfor %}
      {% endif %}
    - extra_hosts:
      - {{ GLOBALS.manager }}:{{ GLOBALS.manager_ip }}
-      {% if DOCKER.containers['so-elastalert'].extra_hosts %}
+      {% if DOCKERMERGED.containers['so-elastalert'].extra_hosts %}
-        {% for XTRAHOST in DOCKER.containers['so-elastalert'].extra_hosts %}
+        {% for XTRAHOST in DOCKERMERGED.containers['so-elastalert'].extra_hosts %}
      - {{ XTRAHOST }}
        {% endfor %}
      {% endif %}
-      {% if DOCKER.containers['so-elastalert'].extra_env %}
+      {% if DOCKERMERGED.containers['so-elastalert'].extra_env %}
    - environment:
-        {% for XTRAENV in DOCKER.containers['so-elastalert'].extra_env %}
+        {% for XTRAENV in DOCKERMERGED.containers['so-elastalert'].extra_env %}
      - {{ XTRAENV }}
        {% endfor %}
      {% endif %}
    {% if DOCKERMERGED.containers['so-elastalert'].ulimits %}
    - ulimits:
    {%   for ULIMIT in DOCKERMERGED.containers['so-elastalert'].ulimits %}
      - {{ ULIMIT.name }}={{ ULIMIT.soft }}:{{ ULIMIT.hard }}
    {%   endfor %}
    {% endif %}
    - require:
      - cmd: wait_for_elasticsearch
      - file: elastarules
@@ -60,7 +66,7 @@ so-elastalert:
    - watch:
      - file: elastaconf
    - onlyif:
-      - "so-elasticsearch-query / | jq -r '.version.number[0:1]' | grep -q 8" {# only run this state if elasticsearch is version 8 #}
+      - "so-elasticsearch-query / | jq -r '.version.number[0:1]' | grep -q 9" {# only run this state if elasticsearch is version 9 #}
 delete_so-elastalert_so-status.disabled:
  file.uncomment:
@@ -1 +0,0 @@
 THIS IS A PLACEHOLDER FILE
@@ -1,47 +1,48 @@
 elastalert:
  enabled:
    description: Enables or disables the ElastAlert 2 process. This process is critical for ensuring alerts arrive in SOC, and for outbound notification delivery.
-    helpLink: elastalert.html
+    forcedType: bool
    helpLink: elastalert
  alerter_parameters:
    title: Custom Configuration Parameters
    description: Optional configuration parameters made available as defaults for all rules and alerters. Use YAML format for these parameters, and reference the ElastAlert 2 documentation, located at https://elastalert2.readthedocs.io, for available configuration parameters. Requires a valid Security Onion license key.
    global: True
    multiline: True
    syntax: yaml
-    helpLink: elastalert.html
+    helpLink: elastalert
    forcedType: string
  jira_api_key:
    title: Jira API Key
    description: Optional configuration parameter for Jira API Key, used instead of the Jira username and password. Requires a valid Security Onion license key.
    global: True
    sensitive: True
-    helpLink: elastalert.html
+    helpLink: elastalert
    forcedType: string
  jira_pass:
    title: Jira Password
    description: Optional configuration parameter for Jira password. Requires a valid Security Onion license key.
    global: True
    sensitive: True
-    helpLink: elastalert.html
+    helpLink: elastalert
    forcedType: string
  jira_user:
    title: Jira Username
    description: Optional configuration parameter for Jira username. Requires a valid Security Onion license key.
    global: True
-    helpLink: elastalert.html
+    helpLink: elastalert
    forcedType: string
  smtp_pass:
    title: SMTP Password
    description: Optional configuration parameter for SMTP password, required for authenticating email servers. Requires a valid Security Onion license key.
    global: True
    sensitive: True
-    helpLink: elastalert.html
+    helpLink: elastalert
    forcedType: string
  smtp_user:
    title: SMTP Username
    description: Optional configuration parameter for SMTP username, required for authenticating email servers. Requires a valid Security Onion license key.
    global: True
-    helpLink: elastalert.html
+    helpLink: elastalert
    forcedType: string
  files:
    custom:
@@ -49,91 +50,131 @@ elastalert:
        description: Optional custom Certificate Authority for connecting to an AlertManager server. To utilize this custom file, the alertmanager_ca_certs key must be set to /opt/elastalert/custom/alertmanager_ca.crt in the Alerter Parameters setting. Requires a valid Security Onion license key.
        global: True
        file: True
-        helpLink: elastalert.html
+        helpLink: elastalert
      gelf_ca__crt:
        description: Optional custom Certificate Authority for connecting to a Graylog server. To utilize this custom file, the graylog_ca_certs key must be set to /opt/elastalert/custom/graylog_ca.crt in the Alerter Parameters setting. Requires a valid Security Onion license key.
        global: True
        file: True
-        helpLink: elastalert.html
+        helpLink: elastalert
      http_post_ca__crt:
        description: Optional custom Certificate Authority for connecting to a generic HTTP server, via the legacy HTTP POST alerter. To utilize this custom file, the http_post_ca_certs key must be set to /opt/elastalert/custom/http_post2_ca.crt in the Alerter Parameters setting. Requires a valid Security Onion license key.
        global: True
        file: True
-        helpLink: elastalert.html
+        helpLink: elastalert
      http_post2_ca__crt:
        description: Optional custom Certificate Authority for connecting to a generic HTTP server, via the newer HTTP POST 2 alerter. To utilize this custom file, the http_post2_ca_certs key must be set to /opt/elastalert/custom/http_post2_ca.crt in the Alerter Parameters setting. Requires a valid Security Onion license key.
        global: True
        file: True
-        helpLink: elastalert.html
+        helpLink: elastalert
      ms_teams_ca__crt:
        description: Optional custom Certificate Authority for connecting to Microsoft Teams server. To utilize this custom file, the ms_teams_ca_certs key must be set to /opt/elastalert/custom/ms_teams_ca.crt in the Alerter Parameters setting. Requires a valid Security Onion license key.
        global: True
        file: True
-        helpLink: elastalert.html
+        helpLink: elastalert
      pagerduty_ca__crt:
        description: Optional custom Certificate Authority for connecting to PagerDuty server. To utilize this custom file, the pagerduty_ca_certs key must be set to /opt/elastalert/custom/pagerduty_ca.crt in the Alerter Parameters setting. Requires a valid Security Onion license key.
        global: True
        file: True
-        helpLink: elastalert.html
+        helpLink: elastalert
      rocket_chat_ca__crt:
        description: Optional custom Certificate Authority for connecting to PagerDuty server. To utilize this custom file, the rocket_chart_ca_certs key must be set to /opt/elastalert/custom/rocket_chat_ca.crt in the Alerter Parameters setting. Requires a valid Security Onion license key.
        global: True
        file: True
-        helpLink: elastalert.html
+        helpLink: elastalert
      smtp__crt:
        description: Optional custom certificate for connecting to an SMTP server. To utilize this custom file, the smtp_cert_file key must be set to /opt/elastalert/custom/smtp.crt in the Alerter Parameters setting. Requires a valid Security Onion license key.
        global: True
        file: True
-        helpLink: elastalert.html
+        helpLink: elastalert
      smtp__key:
        description: Optional custom certificate key for connecting to an SMTP server. To utilize this custom file, the smtp_key_file key must be set to /opt/elastalert/custom/smtp.key in the Alerter Parameters setting. Requires a valid Security Onion license key.
        global: True
        file: True
-        helpLink: elastalert.html
+        helpLink: elastalert
      slack_ca__crt:
        description: Optional custom Certificate Authority for connecting to Slack. To utilize this custom file, the slack_ca_certs key must be set to /opt/elastalert/custom/slack_ca.crt in the Alerter Parameters setting. Requires a valid Security Onion license key.
        global: True
        file: True
-        helpLink: elastalert.html
+        helpLink: elastalert
  config:
    scan_subdirectories:
      description: Recursively scan subdirectories for rules.
      forcedType: bool
      advanced: True
      global: True
      helpLink: elastalert
    disable_rules_on_error:
      description: Disable rules on failure.
      forcedType: bool
      global: True
-      helpLink: elastalert.html
+      helpLink: elastalert
    run_every:
      minutes:
        description: Amount of time in minutes between searches.
        global: True
-        helpLink: elastalert.html
+        helpLink: elastalert
    buffer_time:
      minutes:
        description: Amount of time in minutes to look through.
        global: True
-        helpLink: elastalert.html
+        helpLink: elastalert
    old_query_limit:
      minutes:
        description: Amount of time in minutes between queries to start at the most recently run query.
        global: True
-        helpLink: elastalert.html
+        helpLink: elastalert
    es_conn_timeout:
      description: Timeout in seconds for connecting to and reading from Elasticsearch.
      global: True
-      helpLink: elastalert.html
+      helpLink: elastalert
    max_query_size:
      description: The maximum number of documents that will be returned from Elasticsearch in a single query.
      global: True
-      helpLink: elastalert.html
+      helpLink: elastalert
    use_ssl:
      description: Use SSL to connect to Elasticsearch.
      forcedType: bool
      advanced: True
      global: True
      helpLink: elastalert
    verify_certs:
      description: Verify TLS certificates when connecting to Elasticsearch.
      forcedType: bool
      advanced: True
      global: True
      helpLink: elastalert
    alert_time_limit:
      days:
        description: The retry window for failed alerts.
        global: True
-        helpLink: elastalert.html
+        helpLink: elastalert
    index_settings:
      shards:
        description: The number of shards for elastalert indices.
        global: True
-        helpLink: elastalert.html
+        helpLink: elastalert
      replicas:
        description: The number of replicas for elastalert indices.
        global: True
-        helpLink: elastalert.html
+        helpLink: elastalert
    logging:
      incremental:
        description: When incremental is false (the default), the logging configuration is applied in full, replacing any existing logging setup. When true, only the level attributes of existing loggers and handlers are updated, leaving the rest of the logging configuration unchanged.
        forcedType: bool
        advanced: True
        global: True
        helpLink: elastalert
      disable_existing_loggers:
        description: Disable existing loggers.
        forcedType: bool
        advanced: True
        global: True
        helpLink: elastalert
      loggers:
        '':
          propagate:
            description: Propagate log messages to parent loggers.
            forcedType: bool
            advanced: True
            global: True
            helpLink: elastalert
@@ -6,7 +6,7 @@
 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
-{%   from 'docker/docker.map.jinja' import DOCKER %}
+{%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 include:
  - elastic-fleet-package-registry.config
@@ -21,30 +21,36 @@ so-elastic-fleet-package-registry:
    - user: 948
    - networks:
      - sobridge:
-        - ipv4_address: {{ DOCKER.containers['so-elastic-fleet-package-registry'].ip }}
+        - ipv4_address: {{ DOCKERMERGED.containers['so-elastic-fleet-package-registry'].ip }}
    - extra_hosts:
        - {{ GLOBALS.hostname }}:{{ GLOBALS.node_ip }}
-        {% if DOCKER.containers['so-elastic-fleet-package-registry'].extra_hosts %}
+        {% if DOCKERMERGED.containers['so-elastic-fleet-package-registry'].extra_hosts %}
-          {% for XTRAHOST in DOCKER.containers['so-elastic-fleet-package-registry'].extra_hosts %}
+          {% for XTRAHOST in DOCKERMERGED.containers['so-elastic-fleet-package-registry'].extra_hosts %}
        - {{ XTRAHOST }}
          {% endfor %}
        {% endif %}
    - port_bindings:
-      {% for BINDING in DOCKER.containers['so-elastic-fleet-package-registry'].port_bindings %}
+      {% for BINDING in DOCKERMERGED.containers['so-elastic-fleet-package-registry'].port_bindings %}
      - {{ BINDING }}
      {% endfor %}
-    {% if DOCKER.containers['so-elastic-fleet-package-registry'].custom_bind_mounts %}
+    {% if DOCKERMERGED.containers['so-elastic-fleet-package-registry'].custom_bind_mounts %}
    - binds:
-        {% for BIND in DOCKER.containers['so-elastic-fleet-package-registry'].custom_bind_mounts %}
+        {% for BIND in DOCKERMERGED.containers['so-elastic-fleet-package-registry'].custom_bind_mounts %}
      - {{ BIND }}
        {% endfor %}
    {% endif %}
-    {% if DOCKER.containers['so-elastic-fleet-package-registry'].extra_env %}
+    {% if DOCKERMERGED.containers['so-elastic-fleet-package-registry'].extra_env %}
    - environment:
-        {% for XTRAENV in DOCKER.containers['so-elastic-fleet-package-registry'].extra_env %}
+        {% for XTRAENV in DOCKERMERGED.containers['so-elastic-fleet-package-registry'].extra_env %}
      - {{ XTRAENV }}
        {% endfor %}
      {% endif %}
    {% if DOCKERMERGED.containers['so-elastic-fleet-package-registry'].ulimits %}
    - ulimits:
    {%   for ULIMIT in DOCKERMERGED.containers['so-elastic-fleet-package-registry'].ulimits %}
      - {{ ULIMIT.name }}={{ ULIMIT.soft }}:{{ ULIMIT.hard }}
    {%   endfor %}
    {% endif %}
 delete_so-elastic-fleet-package-registry_so-status.disabled:
  file.uncomment:
    - name: /opt/so/conf/so-status/so-status.conf
@@ -1,4 +1,5 @@
 elastic_fleet_package_registry:
  enabled:
    description: Enables or disables the Fleet package registry process. This process must remain enabled to allow Elastic Agent packages to be updated.
    forcedType: bool
    advanced: True
@@ -6,9 +6,10 @@
 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
-{%   from 'docker/docker.map.jinja' import DOCKER %}
+{%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 include:
  - ca
  - elasticagent.config
  - elasticagent.sostatus
@@ -21,17 +22,17 @@ so-elastic-agent:
    - user: 949
    - networks:
      - sobridge:
-        - ipv4_address: {{ DOCKER.containers['so-elastic-agent'].ip }}
+        - ipv4_address: {{ DOCKERMERGED.containers['so-elastic-agent'].ip }}
    - extra_hosts:
        - {{ GLOBALS.manager }}:{{ GLOBALS.manager_ip }}
        - {{ GLOBALS.hostname }}:{{ GLOBALS.node_ip }}
-        {% if DOCKER.containers['so-elastic-agent'].extra_hosts %}
+        {% if DOCKERMERGED.containers['so-elastic-agent'].extra_hosts %}
-          {% for XTRAHOST in DOCKER.containers['so-elastic-agent'].extra_hosts %}
+          {% for XTRAHOST in DOCKERMERGED.containers['so-elastic-agent'].extra_hosts %}
        - {{ XTRAHOST }}
          {% endfor %}
        {% endif %}
    - port_bindings:
-      {% for BINDING in DOCKER.containers['so-elastic-agent'].port_bindings %}
+      {% for BINDING in DOCKERMERGED.containers['so-elastic-agent'].port_bindings %}
      - {{ BINDING }}
      {% endfor %}
    - binds:
@@ -40,23 +41,31 @@ so-elastic-agent:
      - /etc/pki/tls/certs/intca.crt:/etc/pki/tls/certs/intca.crt:ro 
      - /nsm:/nsm:ro
      - /opt/so/log:/opt/so/log:ro
-     {% if DOCKER.containers['so-elastic-agent'].custom_bind_mounts %}
+     {% if DOCKERMERGED.containers['so-elastic-agent'].custom_bind_mounts %}
-        {% for BIND in DOCKER.containers['so-elastic-agent'].custom_bind_mounts %}
+        {% for BIND in DOCKERMERGED.containers['so-elastic-agent'].custom_bind_mounts %}
      - {{ BIND }}
        {% endfor %}
      {% endif %}
    - environment:
      - FLEET_CA=/etc/pki/tls/certs/intca.crt
      - LOGS_PATH=logs
-      {% if DOCKER.containers['so-elastic-agent'].extra_env %}
+      {% if DOCKERMERGED.containers['so-elastic-agent'].extra_env %}
-        {% for XTRAENV in DOCKER.containers['so-elastic-agent'].extra_env %}
+        {% for XTRAENV in DOCKERMERGED.containers['so-elastic-agent'].extra_env %}
      - {{ XTRAENV }}
        {% endfor %}
      {% endif %}
    {% if DOCKERMERGED.containers['so-elastic-agent'].ulimits %}
    - ulimits:
    {%   for ULIMIT in DOCKERMERGED.containers['so-elastic-agent'].ulimits %}
      - {{ ULIMIT.name }}={{ ULIMIT.soft }}:{{ ULIMIT.hard }}
    {%   endfor %}
    {% endif %}
    - require:
      - file: create-elastic-agent-config
      - file: trusttheca
    - watch:
      - file: create-elastic-agent-config
      - file: trusttheca
 delete_so-elastic-agent_so-status.disabled:
  file.uncomment:
@@ -3,7 +3,7 @@
 {%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %}
 id: aea1ba80-1065-11ee-a369-97538913b6a9
-revision: 1
+revision: 4
 outputs:
  default:
    type: elasticsearch
@@ -22,242 +22,133 @@ agent:
    metrics: false
  features: {}
 inputs:
-  - id: logfile-logs-fefef78c-422f-4cfa-8abf-4cd1b9428f62
+  - id: filestream-filestream-85820eb0-25ef-11f0-a18d-1b26f69b8310
    name: import-evtx-logs
    revision: 2
    type: logfile
    use_output: default
    meta:
      package:
        name: log
        version:
    data_stream:
      namespace: so
    package_policy_id: fefef78c-422f-4cfa-8abf-4cd1b9428f62
    streams:
      - id: logfile-log.log-fefef78c-422f-4cfa-8abf-4cd1b9428f62
        data_stream:
          dataset: import
        paths:
          - /nsm/import/*/evtx/*.json
        processors:
          - dissect:
              field: log.file.path
              tokenizer: '/nsm/import/%{import.id}/evtx/%{import.file}'
              target_prefix: ''
          - decode_json_fields:
              fields:
                - message
              target: ''
          - drop_fields:
              ignore_missing: true
              fields:
                - host
          - add_fields:
              fields:
                dataset: system.security
                type: logs
                namespace: default
              target: data_stream
          - add_fields:
              fields:
                dataset: system.security
                module: system
                imported: true
              target: event
          - then:
              - add_fields:
                  fields:
                    dataset: windows.sysmon_operational
                  target: data_stream
              - add_fields:
                  fields:
                    dataset: windows.sysmon_operational
                    module: windows
                    imported: true
                  target: event
            if:
              equals:
                winlog.channel: Microsoft-Windows-Sysmon/Operational
          - then:
              - add_fields:
                  fields:
                    dataset: system.application
                  target: data_stream
              - add_fields:
                  fields:
                    dataset: system.application
                  target: event
            if:
              equals:
                winlog.channel: Application
          - then:
              - add_fields:
                  fields:
                    dataset: system.system
                  target: data_stream
              - add_fields:
                  fields:
                    dataset: system.system
                  target: event
            if:
              equals:
                winlog.channel: System
          - then:
              - add_fields:
                  fields:
                    dataset: windows.powershell_operational
                  target: data_stream
              - add_fields:
                  fields:
                    dataset: windows.powershell_operational
                    module: windows
                  target: event
            if:
              equals:
                winlog.channel: Microsoft-Windows-PowerShell/Operational
        tags:
          - import
  - id: logfile-redis-fc98c947-7d17-4861-a318-7ad075f6d1b0
    name: redis-logs
    revision: 2
    type: logfile
    use_output: default
    meta:
      package:
        name: redis
        version:
    data_stream:
      namespace: default
    package_policy_id: fc98c947-7d17-4861-a318-7ad075f6d1b0
    streams:
      - id: logfile-redis.log-fc98c947-7d17-4861-a318-7ad075f6d1b0
        data_stream:
          dataset: redis.log
          type: logs
        exclude_files:
          - .gz$
        paths:
          - /opt/so/log/redis/redis.log
        tags:
          - redis-log
        exclude_lines:
          - '^\s+[\-`(''.|_]'
  - id: logfile-logs-3b56803d-5ade-4c93-b25e-9b37182f66b8
    name: import-suricata-logs
-    revision: 2
+    revision: 3
-    type: logfile
+    type: filestream
    use_output: default
    meta:
      package:
-        name: log
+        name: filestream
        version:
    data_stream:
      namespace: so
-    package_policy_id: 3b56803d-5ade-4c93-b25e-9b37182f66b8
+    package_policy_id: 85820eb0-25ef-11f0-a18d-1b26f69b8310
    streams:
-      - id: logfile-log.log-3b56803d-5ade-4c93-b25e-9b37182f66b8
+      - id: filestream-filestream.generic-85820eb0-25ef-11f0-a18d-1b26f69b8310
        data_stream:
          dataset: import
        pipeline: suricata.common
        paths:
          - /nsm/import/*/suricata/eve*.json
        pipeline: suricata.common
        prospector.scanner.recursive_glob: true
        prospector.scanner.exclude_files:
          - \.gz$
        ignore_older: 72h
        clean_inactive: -1
        parsers: null
        processors:
          - add_fields:
              target: event
              fields:
                category: network
                module: suricata
                imported: true
                category: network
              target: event
          - dissect:
              tokenizer: /nsm/import/%{import.id}/suricata/%{import.file}
              field: log.file.path
              tokenizer: '/nsm/import/%{import.id}/suricata/%{import.file}'
              target_prefix: ''
-  - id: logfile-logs-c327e1a3-1ebe-449c-a8eb-f6f35032e69d
+        file_identity.native: null
-    name: soc-server-logs
+        prospector.scanner.fingerprint.enabled: false
-    revision: 2
+  - id: filestream-filestream-86b4e960-25ef-11f0-a18d-1b26f69b8310
-    type: logfile
+    name: import-zeek-logs
    revision: 3
    type: filestream
    use_output: default
    meta:
      package:
-        name: log
+        name: filestream
        version:
    data_stream:
      namespace: so
-    package_policy_id: c327e1a3-1ebe-449c-a8eb-f6f35032e69d
+    package_policy_id: 86b4e960-25ef-11f0-a18d-1b26f69b8310
    streams:
-      - id: logfile-log.log-c327e1a3-1ebe-449c-a8eb-f6f35032e69d
+      - id: filestream-filestream.generic-86b4e960-25ef-11f0-a18d-1b26f69b8310
        data_stream:
-          dataset: soc
+          dataset: import
        pipeline: common
        paths:
-          - /opt/so/log/soc/sensoroni-server.log
+          - /nsm/import/*/zeek/logs/*.log
        prospector.scanner.recursive_glob: true
        prospector.scanner.exclude_files:
          - >-
            (broker|capture_loss|cluster|conn-summary|console|ecat_arp_info|known_certs|known_hosts|known_services|loaded_scripts|ntp|ocsp|packet_filter|reporter|stats|stderr|stdout).log$
        clean_inactive: -1
        parsers: null
        processors:
-          - decode_json_fields:
+          - dissect:
-              add_error_key: true
+              tokenizer: /nsm/import/%{import.id}/zeek/logs/%{import.file}
-              process_array: true
+              field: log.file.path
-              max_depth: 2
+              target_prefix: ''
-              fields:
+          - script:
-                - message
+              lang: javascript
-              target: soc
+              source: |
                function process(event) {
                  var pl = event.Get("import.file").slice(0,-4);
                  event.Put("@metadata.pipeline", "zeek." + pl);
                }
          - add_fields:
              fields:
                module: soc
                dataset_temp: server
                category: host
              target: event
          - rename:
              ignore_missing: true
              fields:
-                - from: soc.fields.sourceIp
+                category: network
-                  to: source.ip
+                module: zeek
-                - from: soc.fields.status
+                imported: true
-                  to: http.response.status_code
+          - add_tags:
-                - from: soc.fields.method
+              tags: ics
-                  to: http.request.method
+              when:
-                - from: soc.fields.path
+                regexp:
-                  to: url.path
+                  import.file: >-
-                - from: soc.message
+                    ^bacnet*|^bsap*|^cip*|^cotp*|^dnp3*|^ecat*|^enip*|^modbus*|^opcua*|^profinet*|^s7comm*
-                  to: event.action
+        file_identity.native: null
-                - from: soc.level
+        prospector.scanner.fingerprint.enabled: false
-                  to: log.level
+  - id: filestream-filestream-91741240-25ef-11f0-a18d-1b26f69b8310
        tags:
          - so-soc
  - id: logfile-logs-906e0d4c-9ec3-4c6a-bef6-e347ec9fd073
    name: soc-sensoroni-logs
-    revision: 2
+    revision: 3
-    type: logfile
+    type: filestream
    use_output: default
    meta:
      package:
-        name: log
+        name: filestream
        version:
    data_stream:
      namespace: so
-    package_policy_id: 906e0d4c-9ec3-4c6a-bef6-e347ec9fd073
+    package_policy_id: 91741240-25ef-11f0-a18d-1b26f69b8310
    streams:
-      - id: logfile-log.log-906e0d4c-9ec3-4c6a-bef6-e347ec9fd073
+      - id: filestream-filestream.generic-91741240-25ef-11f0-a18d-1b26f69b8310
        data_stream:
          dataset: soc
        pipeline: common
        paths:
          - /opt/so/log/sensoroni/sensoroni.log
        pipeline: common
        prospector.scanner.recursive_glob: true
        prospector.scanner.exclude_files:
          - \.gz$
        clean_inactive: -1
        parsers: null
        processors:
          - decode_json_fields:
              add_error_key: true
              process_array: true
              max_depth: 2
              fields:
                - message
              target: sensoroni
              process_array: true
              max_depth: 2
              add_error_key: true
          - add_fields:
              target: event
              fields:
                category: host
                module: soc
                dataset_temp: sensoroni
                category: host
              target: event
          - rename:
              ignore_missing: true
              fields:
                - from: sensoroni.fields.sourceIp
                  to: source.ip
@@ -271,141 +162,100 @@ inputs:
                  to: event.action
                - from: sensoroni.level
                  to: log.level
-  - id: logfile-logs-df0d7f2c-221f-433b-b18b-d1cf83250515
+              ignore_missing: true
-    name: soc-salt-relay-logs
+        file_identity.native: null
-    revision: 2
+        prospector.scanner.fingerprint.enabled: false
-    type: logfile
+  - id: filestream-filestream-976e3900-25ef-11f0-a18d-1b26f69b8310
    use_output: default
    meta:
      package:
        name: log
        version:
    data_stream:
      namespace: so
    package_policy_id: df0d7f2c-221f-433b-b18b-d1cf83250515
    streams:
      - id: logfile-log.log-df0d7f2c-221f-433b-b18b-d1cf83250515
        data_stream:
          dataset: soc
        pipeline: common
        paths:
          - /opt/so/log/soc/salt-relay.log
        processors:
          - dissect:
              field: message
              tokenizer: '%{soc.ts} | %{event.action}'
              target_prefix: ''
          - add_fields:
              fields:
                module: soc
                dataset_temp: salt_relay
                category: host
              target: event
        tags:
          - so-soc
  - id: logfile-logs-74bd2366-fe52-493c-bddc-843a017fc4d0
    name: soc-auth-sync-logs
    revision: 2
    type: logfile
    use_output: default
    meta:
      package:
        name: log
        version:
    data_stream:
      namespace: so
    package_policy_id: 74bd2366-fe52-493c-bddc-843a017fc4d0
    streams:
      - id: logfile-log.log-74bd2366-fe52-493c-bddc-843a017fc4d0
        data_stream:
          dataset: soc
        pipeline: common
        paths:
          - /opt/so/log/soc/sync.log
        processors:
          - dissect:
              field: message
              tokenizer: '%{event.action}'
              target_prefix: ''
          - add_fields:
              fields:
                module: soc
                dataset_temp: auth_sync
                category: host
              target: event
        tags:
          - so-soc
  - id: logfile-logs-d151d9bf-ff2a-4529-9520-c99244bc0253
    name: suricata-logs
-    revision: 2
+    revision: 3
-    type: logfile
+    type: filestream
    use_output: default
    meta:
      package:
-        name: log
+        name: filestream
        version:
    data_stream:
      namespace: so
-    package_policy_id: d151d9bf-ff2a-4529-9520-c99244bc0253
+    package_policy_id: 976e3900-25ef-11f0-a18d-1b26f69b8310
    streams:
-      - id: logfile-log.log-d151d9bf-ff2a-4529-9520-c99244bc0253
+      - id: filestream-filestream.generic-976e3900-25ef-11f0-a18d-1b26f69b8310
        data_stream:
          dataset: suricata
        pipeline: suricata.common
        paths:
          - /nsm/suricata/eve*.json
        pipeline: suricata.common
        prospector.scanner.recursive_glob: true
        prospector.scanner.exclude_files:
          - \.gz$
        clean_inactive: -1
        parsers: null
        processors:
          - add_fields:
              fields:
                module: suricata
                category: network
              target: event
-  - id: logfile-logs-31f94d05-ae75-40ee-b9c5-0e0356eff327
+              fields:
                category: network
                module: suricata
        file_identity.native: null
        prospector.scanner.fingerprint.enabled: false
  - id: filestream-filestream-95091fe0-25ef-11f0-a18d-1b26f69b8310
    name: strelka-logs
-    revision: 2
+    revision: 3
-    type: logfile
+    type: filestream
    use_output: default
    meta:
      package:
-        name: log
+        name: filestream
        version:
    data_stream:
      namespace: so
-    package_policy_id: 31f94d05-ae75-40ee-b9c5-0e0356eff327
+    package_policy_id: 95091fe0-25ef-11f0-a18d-1b26f69b8310
    streams:
-      - id: logfile-log.log-31f94d05-ae75-40ee-b9c5-0e0356eff327
+      - id: filestream-filestream.generic-95091fe0-25ef-11f0-a18d-1b26f69b8310
        data_stream:
          dataset: strelka
        pipeline: strelka.file
        paths:
          - /nsm/strelka/log/strelka.log
        pipeline: strelka.file
        prospector.scanner.recursive_glob: true
        prospector.scanner.exclude_files:
          - \.gz$
        clean_inactive: -1
        parsers: null
        processors:
          - add_fields:
              fields:
                module: strelka
                category: file
              target: event
-  - id: logfile-logs-6197fe84-9b58-4d9b-8464-3d517f28808d
+              fields:
                category: file
                module: strelka
        file_identity.native: null
        prospector.scanner.fingerprint.enabled: false
  - id: filestream-filestream-9f309ca0-25ef-11f0-a18d-1b26f69b8310
    name: zeek-logs
-    revision: 1
+    revision: 2
-    type: logfile
+    type: filestream
    use_output: default
    meta:
      package:
-        name: log
+        name: filestream
-        version: 
+        version:
    data_stream:
      namespace: so
-    package_policy_id: 6197fe84-9b58-4d9b-8464-3d517f28808d
+    package_policy_id: 9f309ca0-25ef-11f0-a18d-1b26f69b8310
    streams:
-      - id: logfile-log.log-6197fe84-9b58-4d9b-8464-3d517f28808d
+      - id: filestream-filestream.generic-9f309ca0-25ef-11f0-a18d-1b26f69b8310
        data_stream:
          dataset: zeek
        paths:
          - /nsm/zeek/logs/current/*.log
        prospector.scanner.recursive_glob: true
        prospector.scanner.exclude_files:
          - >-
            (broker|capture_loss|cluster|conn-summary|console|ecat_arp_info|known_certs|known_hosts|known_services|loaded_scripts|ntp|ocsp|packet_filter|reporter|stats|stderr|stdout).log$
        clean_inactive: -1
        parsers: null
        processors:
          - dissect:
-              tokenizer: '/nsm/zeek/logs/current/%{pipeline}.log'
+              tokenizer: /nsm/zeek/logs/current/%{pipeline}.log
              field: log.file.path
              trim_chars: .log
              target_prefix: ''
@@ -427,18 +277,17 @@ inputs:
                regexp:
                  pipeline: >-
                    ^bacnet*|^bsap*|^cip*|^cotp*|^dnp3*|^ecat*|^enip*|^modbus*|^opcua*|^profinet*|^s7comm*
-        exclude_files:
+        file_identity.native: null
-          - >-
+        prospector.scanner.fingerprint.enabled: false
            broker|capture_loss|cluster|ecat_arp_info|known_hosts|known_services|loaded_scripts|ntp|ocsp|packet_filter|reporter|stats|stderr|stdout.log$
  - id: udp-udp-35051de0-46a5-11ee-8d5d-9f98c8182f60
    name: syslog-udp-514
-    revision: 3
+    revision: 4
    type: udp
    use_output: default
    meta:
      package:
        name: udp
-        version: 1.10.0
+        version:
    data_stream:
      namespace: so
    package_policy_id: 35051de0-46a5-11ee-8d5d-9f98c8182f60
@@ -458,13 +307,13 @@ inputs:
          - syslog
  - id: tcp-tcp-33d37bb0-46a5-11ee-8d5d-9f98c8182f60
    name: syslog-tcp-514
-    revision: 3
+    revision: 4
    type: tcp
    use_output: default
    meta:
      package:
        name: tcp
-        version: 1.10.0
+        version:
    data_stream:
      namespace: so
    package_policy_id: 33d37bb0-46a5-11ee-8d5d-9f98c8182f60
@@ -1,4 +1,5 @@
 elasticagent:
  enabled:
    description: Enables or disables the Elastic Agent process. This process must remain enabled to allow collection of node events.
    forcedType: bool
    advanced: True
@@ -0,0 +1,34 @@
 {# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
   or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
   https://securityonion.net/license; you may not use this file except in compliance with the
   Elastic License 2.0. #}
 {% from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %}
 {# advanced config_yaml options for elasticfleet logstash output #}
 {% set ADV_OUTPUT_LOGSTASH_RAW = ELASTICFLEETMERGED.config.outputs.logstash %}
 {% set ADV_OUTPUT_LOGSTASH = {} %}
 {% for k, v in ADV_OUTPUT_LOGSTASH_RAW.items() %}
 {%   if v != "" and v is not none %}
 {%     if k == 'queue_mem_events' %}
 {#       rename queue_mem_events queue.mem.events #}
 {%       do ADV_OUTPUT_LOGSTASH.update({'queue.mem.events':v}) %}
 {%     elif k == 'loadbalance' %}
 {%       if v %}
 {#         only include loadbalance config when its True #}
 {%         do ADV_OUTPUT_LOGSTASH.update({k:v}) %}
 {%       endif %}
 {%     else %}
 {%       do ADV_OUTPUT_LOGSTASH.update({k:v}) %}
 {%     endif %}
 {%   endif %}
 {% endfor %}
 {% set LOGSTASH_CONFIG_YAML_RAW = [] %}
 {% if ADV_OUTPUT_LOGSTASH %}
 {%   for k, v in ADV_OUTPUT_LOGSTASH.items() %}
 {%     do LOGSTASH_CONFIG_YAML_RAW.append(k ~ ': ' ~ v) %}
 {%   endfor %}
 {% endif %}
 {% set LOGSTASH_CONFIG_YAML = LOGSTASH_CONFIG_YAML_RAW | join('\\n') if LOGSTASH_CONFIG_YAML_RAW else '' %}
@@ -11,6 +11,7 @@
 include:
  - elasticfleet.artifact_registry
  - elasticfleet.ssl
 # Add EA Group
 elasticfleetgroup:
@@ -95,6 +96,9 @@ soresourcesrepoclone:
    - rev: 'main'
    - depth: 1
    - force_reset: True
    - retry:
        attempts: 3
        interval: 10
 {% endif %}
 elasticdefendconfdir:
@@ -0,0 +1,123 @@
 {# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
  or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use
  this file except in compliance with the Elastic License 2.0. #}
 {% import_json '/opt/so/state/esfleet_content_package_components.json' as ADDON_CONTENT_PACKAGE_COMPONENTS %}
 {% import_json '/opt/so/state/esfleet_component_templates.json' as INSTALLED_COMPONENT_TEMPLATES %}
 {% import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %}
 {% set CORE_ESFLEET_PACKAGES = ELASTICFLEETDEFAULTS.get('elasticfleet', {}).get('packages', {}) %}
 {% set ADDON_CONTENT_INTEGRATION_DEFAULTS = {} %}
 {% set DEBUG_STUFF = {} %}
 {% for pkg in ADDON_CONTENT_PACKAGE_COMPONENTS %}
 {%   if pkg.name in CORE_ESFLEET_PACKAGES %}
 {#     skip core content packages #}
 {%   elif pkg.name not in CORE_ESFLEET_PACKAGES %}
 {#     generate defaults for each content package #}
 {%     if pkg.dataStreams is defined and pkg.dataStreams is not none and pkg.dataStreams | length > 0%}
 {%       for pattern in pkg.dataStreams %}
 {#         in ES 9.3.2 'input' type integrations no longer create default component templates and instead they wait for user input during 'integration' setup (fleet ui config)
             title: generic is an artifact of that and is not in use #}
 {%         if pattern.title == "generic" %}
 {%           continue %}
 {%         endif %}
 {%         if "metrics-" in pattern.name %}
 {%           set integration_type = "metrics-" %}
 {%         elif "logs-" in pattern.name %}
 {%           set integration_type = "logs-" %}
 {%         else %}
 {%           set integration_type = "" %}
 {%         endif %}
 {#         on content integrations the component name is user defined at the time it is added to an agent policy #}
 {%         set component_name = pattern.title %}
 {%         set index_pattern = pattern.name %}
 {#           component_name_x maintains the functionality of merging local pillar changes with generated 'defaults' via SOC UI #}
 {%           set component_name_x = component_name.replace(".","_x_") %}
 {#           pillar overrides/merge expects the key names to follow the naming in elasticsearch/defaults.yaml eg. so-logs-1password_x_item_usages . The _x_ is replaced later on in elasticsearch/template.map.jinja #}
 {%           set integration_key = "so-" ~ integration_type ~ pkg.name + '_x_' ~ component_name_x %}
 {#           Default integration settings #}
 {%           set integration_defaults = {
               "index_sorting": false,
               "index_template": {
                 "composed_of": [integration_type ~ component_name ~ "@package", integration_type ~ component_name ~ "@custom", "so-fleet_integrations.ip_mappings-1", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"],
                 "data_stream": {
                   "allow_custom_routing": false,
                   "hidden": false
                 },
                 "ignore_missing_component_templates": [integration_type ~ component_name ~ "@custom"],
                 "index_patterns": [index_pattern],
                 "priority": 501,
                 "template": {
                   "settings": {
                     "index": {
                       "lifecycle": {"name": "so-" ~ integration_type ~ component_name ~ "-logs"},
                         "number_of_replicas": 0
                     }
                   }
                 }
               },
               "policy": {
                 "phases": {
                   "cold": {
                     "actions": {
                        "allocate":{
                          "number_of_replicas": ""
                        },
                       "set_priority": {"priority": 0}
                     },
                     "min_age": "60d"
                   },
                   "delete": {
                     "actions": {
                       "delete": {}
                     },
                     "min_age": "365d"
                   },
                   "hot": {
                     "actions": {
                       "rollover": {
                         "max_age": "30d",
                         "max_primary_shard_size": "50gb"
                       },
                       "forcemerge":{
                         "max_num_segments": ""
                       },
                       "shrink":{
                         "max_primary_shard_size": "",
                         "method": "COUNT",
                         "number_of_shards": ""
                       },
                       "set_priority": {"priority": 100}
                      },
                      "min_age": "0ms"
                   },
                   "warm": {
                     "actions": {
                        "allocate": {
                          "number_of_replicas": ""
                        },
                        "forcemerge": {
                          "max_num_segments": ""
                        },
                        "shrink":{
                          "max_primary_shard_size": "",
                          "method": "COUNT",
                          "number_of_shards": ""
                        },
                       "set_priority": {"priority": 50}
                     },
                     "min_age": "30d"
                   }
                 }
               }
             } %}
 {%         do ADDON_CONTENT_INTEGRATION_DEFAULTS.update({integration_key: integration_defaults}) %}
 {%       endfor %}
 {%     else %}
 {%     endif %}
 {%   endif %}
 {% endfor %}
@@ -1,5 +1,6 @@
 elasticfleet:
  enabled: False
  patch_version: 9.3.3+build202604082258 # Elastic Agent specific patch release.
  enable_manager_output: True
  config:
    server:
@@ -10,12 +11,19 @@ elasticfleet:
      grid_enrollment: ''
    defend_filters:
      enable_auto_configuration: False
    outputs:
      logstash:
        bulk_max_size: ''
        worker: ''
        queue_mem_events: ''
        timeout: ''
        loadbalance: False
        compression_level: ''
    subscription_integrations: False
    auto_upgrade_integrations: False
  logging:
    zeek:
      excluded:
        - analyzer
        - broker
        - capture_loss
        - cluster
@@ -6,16 +6,17 @@
 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
-{%   from 'docker/docker.map.jinja' import DOCKER %}
+{%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {%   from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %}
 {#   This value is generated during node install and stored in minion pillar #}
 {%   set SERVICETOKEN = salt['pillar.get']('elasticfleet:config:server:es_token','') %}
 include:
  - ca
  - logstash.ssl
  - elasticfleet.config
  - elasticfleet.sostatus
  - ssl
 {% if grains.role not in ['so-fleet'] %}
 # Wait for Elasticsearch to be ready - no reason to try running Elastic Fleet server if ES is not ready
@@ -32,6 +33,17 @@ so-elastic-fleet-auto-configure-logstash-outputs:
    - retry:
        attempts: 4
        interval: 30
 {# Separate from above in order to catch elasticfleet-logstash.crt changes and force update to fleet output policy #}
 so-elastic-fleet-auto-configure-logstash-outputs-force:
  cmd.run:
    - name: /usr/sbin/so-elastic-fleet-outputs-update --certs
    - retry:
        attempts: 4
        interval: 30
    - onchanges:
        - x509: etc_elasticfleet_logstash_crt
        - x509: elasticfleet_kafka_crt
 {% endif %}
 # If enabled, automatically update Fleet Server URLs & ES Connection
@@ -82,17 +94,17 @@ so-elastic-fleet:
    - user: 947
    - networks:
      - sobridge:
-        - ipv4_address: {{ DOCKER.containers['so-elastic-fleet'].ip }}
+        - ipv4_address: {{ DOCKERMERGED.containers['so-elastic-fleet'].ip }}
    - extra_hosts:
        - {{ GLOBALS.manager }}:{{ GLOBALS.manager_ip }}
        - {{ GLOBALS.hostname }}:{{ GLOBALS.node_ip }}
-        {% if DOCKER.containers['so-elastic-fleet'].extra_hosts %}
+        {% if DOCKERMERGED.containers['so-elastic-fleet'].extra_hosts %}
-          {% for XTRAHOST in DOCKER.containers['so-elastic-fleet'].extra_hosts %}
+          {% for XTRAHOST in DOCKERMERGED.containers['so-elastic-fleet'].extra_hosts %}
        - {{ XTRAHOST }}
          {% endfor %}
        {% endif %}
    - port_bindings:
-      {% for BINDING in DOCKER.containers['so-elastic-fleet'].port_bindings %}
+      {% for BINDING in DOCKERMERGED.containers['so-elastic-fleet'].port_bindings %}
      - {{ BINDING }}
      {% endfor %}
    - binds:
@@ -100,8 +112,8 @@ so-elastic-fleet:
      - /etc/pki/elasticfleet-server.key:/etc/pki/elasticfleet-server.key:ro
      - /etc/pki/tls/certs/intca.crt:/etc/pki/tls/certs/intca.crt:ro
      - /opt/so/log/elasticfleet:/usr/share/elastic-agent/logs 
-     {% if DOCKER.containers['so-elastic-fleet'].custom_bind_mounts %}
+     {% if DOCKERMERGED.containers['so-elastic-fleet'].custom_bind_mounts %}
-        {% for BIND in DOCKER.containers['so-elastic-fleet'].custom_bind_mounts %}
+        {% for BIND in DOCKERMERGED.containers['so-elastic-fleet'].custom_bind_mounts %}
      - {{ BIND }}
        {% endfor %}
      {% endif %}      
@@ -116,12 +128,23 @@ so-elastic-fleet:
      - FLEET_CA=/etc/pki/tls/certs/intca.crt     
      - FLEET_SERVER_ELASTICSEARCH_CA=/etc/pki/tls/certs/intca.crt
      - LOGS_PATH=logs
-      {% if DOCKER.containers['so-elastic-fleet'].extra_env %}
+      {% if DOCKERMERGED.containers['so-elastic-fleet'].extra_env %}
-        {% for XTRAENV in DOCKER.containers['so-elastic-fleet'].extra_env %}
+        {% for XTRAENV in DOCKERMERGED.containers['so-elastic-fleet'].extra_env %}
      - {{ XTRAENV }}
        {% endfor %}
      {% endif %}
    {% if DOCKERMERGED.containers['so-elastic-fleet'].ulimits %}
    - ulimits:
    {%   for ULIMIT in DOCKERMERGED.containers['so-elastic-fleet'].ulimits %}
      - {{ ULIMIT.name }}={{ ULIMIT.soft }}:{{ ULIMIT.hard }}
    {%   endfor %}
    {% endif %}
    - watch:
      - file: trusttheca
      - x509: etc_elasticfleet_key
      - x509: etc_elasticfleet_crt
    - require:
      - file: trusttheca
      - x509: etc_elasticfleet_key
      - x509: etc_elasticfleet_crt
 {%   endif %}
@@ -135,12 +158,18 @@ so-elastic-fleet-package-statefile:
 so-elastic-fleet-package-upgrade:
  cmd.run:
    - name: /usr/sbin/so-elastic-fleet-package-upgrade
    - retry:
        attempts: 3
        interval: 10
    - onchanges:
      - file: /opt/so/state/elastic_fleet_packages.txt
 so-elastic-fleet-integrations:
  cmd.run:
    - name: /usr/sbin/so-elastic-fleet-integration-policy-load
    - retry:
        attempts: 3
        interval: 10
 so-elastic-agent-grid-upgrade:
  cmd.run:
@@ -152,7 +181,11 @@ so-elastic-agent-grid-upgrade:
 so-elastic-fleet-integration-upgrade:
  cmd.run:
    - name: /usr/sbin/so-elastic-fleet-integration-upgrade
    - retry:
        attempts: 3
        interval: 10
 {# Optional integrations script doesn't need the retries like so-elastic-fleet-integration-upgrade which loads the default integrations #}
 so-elastic-fleet-addon-integrations:
  cmd.run:
    - name: /usr/sbin/so-elastic-fleet-optional-integrations-load
@@ -2,27 +2,46 @@
 {%- raw -%}
 {
  "package": {
-    "name": "log",
+    "name": "filestream",
    "version": ""
  },
  "name": "import-zeek-logs",
  "namespace": "so",
  "description": "Zeek Import logs",
  "policy_id": "so-grid-nodes_general",
  "policy_ids": [
    "so-grid-nodes_general"
  ],
  "vars": {},
  "inputs": {
-    "logs-logfile": {
+    "filestream-filestream": {
      "enabled": true,
      "streams": {
-        "log.logs": {
+        "filestream.filestream": {
          "enabled": true,
          "vars": {
            "paths": [
              "/nsm/import/*/zeek/logs/*.log"
            ],
            "compression_gzip": false,
            "use_logs_stream": false,
            "data_stream.dataset": "import",
-            "tags": [],
+            "pipeline": "",
            "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
            "exclude_files": ["({%- endraw -%}{{ ELASTICFLEETMERGED.logging.zeek.excluded | join('|') }}{%- raw -%})(\\..+)?\\.log$"],
            "include_files": [],
            "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/zeek/logs/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- script:\n      lang: javascript\n      source: >\n        function process(event) {\n          var pl = event.Get(\"import.file\").slice(0,-4);\n          event.Put(\"@metadata.pipeline\", \"zeek.\" + pl);\n        }\n- add_fields:\n    target: event\n    fields:\n      category: network\n      module: zeek\n      imported: true\n- add_tags:\n    tags: \"ics\"\n    when:\n      regexp:\n        import.file: \"^bacnet*|^bsap*|^cip*|^cotp*|^dnp3*|^ecat*|^enip*|^modbus*|^opcua*|^profinet*|^s7comm*\"",
-            "custom": "exclude_files: [\"{%- endraw -%}{{ ELASTICFLEETMERGED.logging.zeek.excluded | join('|') }}{%- raw -%}.log$\"]\n"
+            "tags": [],
            "recursive_glob": true,
            "clean_inactive": -1,
            "harvester_limit": 0,
            "fingerprint": false,
            "fingerprint_offset": 0,
            "fingerprint_length": "64",
            "file_identity_native": true,
            "exclude_lines": [],
            "include_lines": [],
            "delete_enabled": false
          }
        }
      }
@@ -11,36 +11,57 @@
 {%- endif -%}
 {
  "package": {
-    "name": "log",
+    "name": "filestream",
    "version": ""
  },
  "name": "kratos-logs",
  "namespace": "so",
  "description": "Kratos logs",
  "policy_id": "so-grid-nodes_general",
  "policy_ids": [
    "so-grid-nodes_general"
  ],
  "vars": {},
  "inputs": {
-    "logs-logfile": {
+    "filestream-filestream": {
      "enabled": true,
      "streams": {
-        "log.logs": {
+        "filestream.filestream": {
          "enabled": true,
          "vars": {
            "paths": [
              "/opt/so/log/kratos/kratos.log"
            ],
            "compression_gzip": false,
            "use_logs_stream": false,
            "data_stream.dataset": "kratos",
-            "tags": ["so-kratos"],
+            "pipeline": "kratos",
            "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
            "exclude_files": [
              "\\.gz$"
            ],
            "include_files": [],
            {%- if valid_identities -%}
            "processors": "- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n    add_error_key: true\n- add_fields:\n    target: event\n    fields:\n      category: iam\n      module: kratos\n- if:\n    has_fields:\n      - identity_id\n  then:{% for id, email in identities %}\n    - if:\n        equals:\n          identity_id: \"{{ id }}\"\n      then:\n        - add_fields:\n            target: ''\n            fields:\n              user.name: \"{{ email }}\"{% endfor %}",
            {%- else -%}
            "processors": "- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n    add_error_key: true\n- add_fields:\n    target: event\n    fields:\n      category: iam\n      module: kratos",
            {%- endif -%}
-            "custom": "pipeline: kratos"
+            "tags": [
              "so-kratos"
            ],
            "recursive_glob": true,
            "clean_inactive": -1,
            "harvester_limit": 0,
            "fingerprint": false,
            "fingerprint_offset": 0,
            "file_identity_native": true,
            "exclude_lines": [],
            "include_lines": [],
            "delete_enabled": false
          }
        }
      }
    }
  },
  "force": true
-}
+}
@@ -2,28 +2,44 @@
 {%- raw -%}
 {
  "package": {
-    "name": "log",
+    "name": "filestream",
    "version": ""
  },
  "id": "zeek-logs",
  "name": "zeek-logs",
  "namespace": "so",
  "description": "Zeek logs",
  "policy_id": "so-grid-nodes_general",
  "policy_ids": [
    "so-grid-nodes_general"
  ],
  "vars": {},
  "inputs": {
-    "logs-logfile": {
+    "filestream-filestream": {
      "enabled": true,
      "streams": {
-        "log.logs": {
+        "filestream.filestream": {
          "enabled": true,
          "vars": {
            "paths": [
              "/nsm/zeek/logs/current/*.log"
            ],
            "compression_gzip": false,
            "use_logs_stream": false,
            "data_stream.dataset": "zeek",
-            "tags": [],
+            "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
            "exclude_files": ["({%- endraw -%}{{ ELASTICFLEETMERGED.logging.zeek.excluded | join('|') }}{%- raw -%})(\\..+)?\\.log$"],
            "include_files": [],
            "processors": "- dissect:\n    tokenizer: \"/nsm/zeek/logs/current/%{pipeline}.log\"\n    field: \"log.file.path\"\n    trim_chars: \".log\"\n    target_prefix: \"\"\n- script:\n      lang: javascript\n      source: >\n        function process(event) {\n          var pl = event.Get(\"pipeline\");\n          event.Put(\"@metadata.pipeline\", \"zeek.\" + pl);\n        }\n- add_fields:\n    target: event\n    fields:\n      category: network\n      module: zeek\n- add_tags:\n    tags: \"ics\"\n    when:\n      regexp:\n        pipeline: \"^bacnet*|^bsap*|^cip*|^cotp*|^dnp3*|^ecat*|^enip*|^modbus*|^opcua*|^profinet*|^s7comm*\"",
-            "custom": "exclude_files: [\"{%- endraw -%}{{ ELASTICFLEETMERGED.logging.zeek.excluded | join('|') }}{%- raw -%}.log$\"]\n"
+            "tags": [],
            "recursive_glob": true,
            "clean_inactive": -1,
            "harvester_limit": 0,
            "fingerprint": false,
            "fingerprint_offset": 0,
            "file_identity_native": true,
            "exclude_lines": [],
            "include_lines": [],
            "delete_enabled": false
          }
        }
      }
@@ -31,4 +47,4 @@
  },
  "force": true
 }
-{%- endraw -%}
+{%- endraw -%}
@@ -5,7 +5,7 @@
  "package": {
    "name": "endpoint",
    "title": "Elastic Defend",
-    "version": "8.18.1",
+    "version": "9.3.0",
    "requires_root": true
  },
  "enabled": true,
@@ -6,21 +6,23 @@
  "name": "agent-monitor",
  "namespace": "",
  "description": "",
  "policy_id": "so-grid-nodes_general",
  "policy_ids": [
    "so-grid-nodes_general"
  ],
  "output_id": null,
  "vars": {},
  "inputs": {
    "filestream-filestream": {
      "enabled": true,
      "streams": {
-        "filestream.generic": {
+        "filestream.filestream": {
          "enabled": true,
          "vars": {
            "paths": [
              "/opt/so/log/agents/agent-monitor.log"
            ],
            "compression_gzip": false,
            "use_logs_stream": false,
            "data_stream.dataset": "agentmonitor",
            "pipeline": "elasticagent.monitor",
            "parsers": "",
@@ -34,15 +36,16 @@
            "ignore_older": "72h",
            "clean_inactive": -1,
            "harvester_limit": 0,
-            "fingerprint": true,
+            "fingerprint": false,
            "fingerprint_offset": 0,
-            "fingerprint_length": 64,
+            "file_identity_native": true,
            "file_identity_native": false,
            "exclude_lines": [],
-            "include_lines": []
+            "include_lines": [],
            "delete_enabled": false
          }
        }
      }
    }
-  }
+  },
  "force": true
 }
@@ -40,7 +40,7 @@
          "enabled": true,
          "vars": {
            "paths": [
-              "/opt/so/log/elasticsearch/*.log"
+              "/opt/so/log/elasticsearch/*.json"
            ]
          }
        },
@@ -1,26 +1,49 @@
 {
  "package": {
-    "name": "log",
+    "name": "filestream",
    "version": ""
  },
  "name": "hydra-logs",
  "namespace": "so",
  "description": "Hydra logs",
  "policy_id": "so-grid-nodes_general",
  "policy_ids": [
    "so-grid-nodes_general"
  ],
  "vars": {},
  "inputs": {
-    "logs-logfile": {
+    "filestream-filestream": {
      "enabled": true,
      "streams": {
-        "log.logs": {
+        "filestream.filestream": {
          "enabled": true,
          "vars": {
            "paths": [
              "/opt/so/log/hydra/hydra.log"
            ],
            "compression_gzip": false,
            "use_logs_stream": false,
            "data_stream.dataset": "hydra",
-            "tags": ["so-hydra"],
+            "pipeline": "hydra",
-            "processors": "- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n    add_error_key: true      \n- add_fields:\n    target: event\n    fields:\n      category: iam\n      module: hydra",
+            "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
-            "custom": "pipeline: hydra"
+            "exclude_files": [
              "\\.gz$"
            ],
            "include_files": [],
            "processors": "- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n    add_error_key: true\n- add_fields:\n    target: event\n    fields:\n      category: iam\n      module: hydra",
            "tags": [
              "so-hydra"
            ],
            "recursive_glob": true,
            "ignore_older": "72h",
            "clean_inactive": -1,
            "harvester_limit": 0,
            "fingerprint": false,
            "fingerprint_offset": 0,
            "file_identity_native": true,
            "exclude_lines": [],
            "include_lines": [],
            "delete_enabled": false
          }
        }
      }
@@ -28,3 +51,5 @@
  },
  "force": true
 }
@@ -1,30 +1,50 @@
 {
  "package": {
-    "name": "log",
+    "name": "filestream",
    "version": ""
  },
  "name": "idh-logs",
  "namespace": "so",
  "description": "IDH integration",
  "policy_id": "so-grid-nodes_general",
  "policy_ids": [
    "so-grid-nodes_general"
  ],
  "vars": {},
  "inputs": {
-    "logs-logfile": {
+    "filestream-filestream": {
      "enabled": true,
      "streams": {
-        "log.logs": {
+        "filestream.filestream": {
          "enabled": true,
          "vars": {
            "paths": [
              "/nsm/idh/opencanary.log"
            ],
            "compression_gzip": false,
            "use_logs_stream": false,
            "data_stream.dataset": "idh",
-            "tags": [],
+            "pipeline": "common",
            "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
            "exclude_files": [
              "\\.gz$"
            ],
            "include_files": [],
            "processors": "\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n    add_error_key: true\n- convert:\n    fields:\n      - {from: \"logtype\", to: \"event.code\", type: \"string\"}\n- drop_fields:\n    when:\n      equals:\n        event.code: \"1001\"\n    fields: [\"src_host\", \"src_port\", \"dst_host\", \"dst_port\" ]\n    ignore_missing: true\n- rename:\n    fields:\n      - from: \"src_host\"\n        to: \"source.ip\"\n      - from: \"src_port\"\n        to: \"source.port\"\n      - from: \"dst_host\"\n        to: \"destination.host\"\n      - from: \"dst_port\"\n        to: \"destination.port\"\n    ignore_missing: true\n- drop_fields:\n    fields: '[\"prospector\", \"input\", \"offset\", \"beat\"]'\n- add_fields:\n    target: event\n    fields:\n      category: host\n      module: opencanary",
-            "custom": "pipeline: common"
+            "tags": [],
            "recursive_glob": true,
            "clean_inactive": -1,
            "harvester_limit": 0,
            "fingerprint": false,
            "fingerprint_offset": 0,
            "file_identity_native": true,
            "exclude_lines": [],
            "include_lines": [],
            "delete_enabled": false
          }
        }
      }
    }
  },
  "force": true
-}
+}
@@ -1,33 +1,52 @@
 {
  "package": {
-    "name": "log",
+    "name": "filestream",
    "version": ""
  },
  "name": "import-evtx-logs",
  "namespace": "so",
  "description": "Import Windows EVTX logs",
  "policy_id": "so-grid-nodes_general",
  "policy_ids": [
    "so-grid-nodes_general"
  ],
  "vars": {},
  "inputs": {
-    "logs-logfile": {
+    "filestream-filestream": {
      "enabled": true,
      "streams": {
-        "log.logs": {
+        "filestream.filestream": {
          "enabled": true,
          "vars": {
            "paths": [
              "/nsm/import/*/evtx/*.json"
            ],
            "compression_gzip": false,
            "use_logs_stream": false,
            "data_stream.dataset": "import",
-            "custom": "",
+            "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
-            "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- drop_fields:\n    fields: [\"host\"]\n    ignore_missing: true\n- add_fields:\n    target: data_stream\n    fields:\n      type: logs\n      dataset: system.security\n- add_fields:\n    target: event\n    fields:\n      dataset: system.security\n      module: system\n      imported: true\n- add_fields:\n    target: \"@metadata\"\n    fields:\n        pipeline: logs-system.security-2.5.4\n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.sysmon_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.sysmon_operational\n          module: windows\n          imported: true\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.sysmon_operational-3.1.2\n- if:\n    equals:\n      winlog.channel: 'Application'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.application-2.5.4\n- if:\n    equals:\n      winlog.channel: 'System'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.system-2.5.4\n    \n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.powershell_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell_operational\n          module: windows\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.powershell_operational-3.1.2\n- add_fields:\n    target: data_stream\n    fields:\n        dataset: import",
+            "exclude_files": [
              "\\.gz$"
            ],
            "include_files": [],
            "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- drop_fields:\n    fields: [\"host\"]\n    ignore_missing: true\n- add_fields:\n    target: data_stream\n    fields:\n      type: logs\n      dataset: system.security\n- add_fields:\n    target: event\n    fields:\n      dataset: system.security\n      module: system\n      imported: true\n- add_fields:\n    target: \"@metadata\"\n    fields:\n        pipeline: logs-system.security-2.15.0\n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.sysmon_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.sysmon_operational\n          module: windows\n          imported: true\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.sysmon_operational-3.8.0\n- if:\n    equals:\n      winlog.channel: 'Application'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.application-2.15.0\n- if:\n    equals:\n      winlog.channel: 'System'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.system-2.15.0\n    \n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.powershell_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell_operational\n          module: windows\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.powershell_operational-3.8.0\n- add_fields:\n    target: data_stream\n    fields:\n        dataset: import",
            "tags": [
              "import"
-            ]
+            ],
            "recursive_glob": true,
            "ignore_older": "72h",
            "clean_inactive": -1,
            "harvester_limit": 0,
            "fingerprint": false,
            "fingerprint_offset": 0,
            "file_identity_native": true,
            "exclude_lines": [],
            "include_lines": [],
            "delete_enabled": false
          }
        }
      }
    }
  },
  "force": true
-}
+}
@@ -1,30 +1,51 @@
 {
  "package": {
-    "name": "log",
+    "name": "filestream",
    "version": ""
  },
  "name": "import-suricata-logs",
  "namespace": "so",
  "description": "Import Suricata logs",
  "policy_id": "so-grid-nodes_general",
  "policy_ids": [
    "so-grid-nodes_general"
  ],
  "vars": {},
  "inputs": {
-    "logs-logfile": {
+    "filestream-filestream": {
      "enabled": true,
      "streams": {
-        "log.logs": {
+        "filestream.filestream": {
          "enabled": true,
          "vars": {
            "paths": [
              "/nsm/import/*/suricata/eve*.json"
            ],
            "compression_gzip": false,
            "use_logs_stream": false,
            "data_stream.dataset": "import",
            "pipeline": "suricata.common",
            "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
            "exclude_files": [
              "\\.gz$"
            ],
            "include_files": [],
            "processors": "- add_fields:\n    target: event\n    fields:\n      category: network\n      module: suricata\n      imported: true\n- dissect:\n      tokenizer: \"/nsm/import/%{import.id}/suricata/%{import.file}\"\n      field: \"log.file.path\"\n      target_prefix: \"\"\n",
            "tags": [],
-            "processors": "- add_fields:\n    target: event\n    fields:\n      category: network\n      module: suricata\n      imported: true\n- dissect:\n      tokenizer: \"/nsm/import/%{import.id}/suricata/%{import.file}\"\n      field: \"log.file.path\"\n      target_prefix: \"\"",
+            "recursive_glob": true,
-            "custom": "pipeline: suricata.common"
+            "ignore_older": "72h",
            "clean_inactive": -1,
            "harvester_limit": 0,
            "fingerprint": false,
            "fingerprint_offset": 0,
            "file_identity_native": true,
            "exclude_lines": [],
            "include_lines": [],
            "delete_enabled": false
          }
        }
      }
    }
  },
  "force": true
-}
+}
@@ -15,7 +15,7 @@
          "enabled": true,
          "vars": {
            "paths": [
-              "/opt/so/log/redis/redis.log"
+              "/opt/so/log/redis/redis-server.log"
            ],
            "tags": [
              "redis-log"
@@ -1,18 +1,21 @@
 {
  "package": {
-    "name": "log",
+    "name": "filestream",
    "version": ""
  },
  "name": "rita-logs",
  "namespace": "so",
  "description": "RITA Logs",
  "policy_id": "so-grid-nodes_general",
  "policy_ids": [
    "so-grid-nodes_general"
  ],
  "vars": {},
  "inputs": {
-    "logs-logfile": {
+    "filestream-filestream": {
      "enabled": true,
      "streams": {
-        "log.logs": {
+        "filestream.filestream": {
          "enabled": true,
          "vars": {
            "paths": [
@@ -20,15 +23,30 @@
              "/nsm/rita/exploded-dns.csv",
              "/nsm/rita/long-connections.csv"
            ],
-            "exclude_files": [],
+            "compression_gzip": false,
-            "ignore_older": "72h",
+            "use_logs_stream": false,
            "data_stream.dataset": "rita",
-            "tags": [],
+            "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
            "exclude_files": [
              "\\.gz$"
            ],
            "include_files": [],
            "processors": "- dissect:\n    tokenizer: \"/nsm/rita/%{pipeline}.csv\"\n    field: \"log.file.path\"\n    trim_chars: \".csv\"\n    target_prefix: \"\"\n- script:\n      lang: javascript\n      source: >\n        function process(event) {\n          var pl = event.Get(\"pipeline\").split(\"-\");\n          if (pl.length > 1) {\n            pl = pl[1];\n          }\n          else {\n            pl = pl[0];\n          }\n          event.Put(\"@metadata.pipeline\", \"rita.\" + pl);\n        }\n- add_fields:\n    target: event\n    fields:\n      category: network\n      module: rita",
-            "custom": "exclude_lines: ['^Score', '^Source', '^Domain', '^No results']"
+            "tags": [],
            "recursive_glob": true,
            "ignore_older": "72h",
            "clean_inactive": -1,
            "harvester_limit": 0,
            "fingerprint": false,
            "fingerprint_offset": 0,
            "file_identity_native": true,
            "exclude_lines": [],
            "include_lines": [],
            "delete_enabled": false
          }
        }
      }
    }
-  }
+  },
  "force": true
 }
@@ -1,29 +1,47 @@
 {
  "package": {
-    "name": "log",
+    "name": "filestream",
    "version": ""
  },
  "name": "so-ip-mappings",
  "namespace": "so",
  "description": "IP Description mappings",
  "policy_id": "so-grid-nodes_general",
  "policy_ids": [
    "so-grid-nodes_general"
  ],
  "vars": {},
  "inputs": {
-    "logs-logfile": {
+    "filestream-filestream": {
      "enabled": true,
      "streams": {
-        "log.logs": {
+        "filestream.filestream": {
          "enabled": true,
          "vars": {
            "paths": [
              "/nsm/custom-mappings/ip-descriptions.csv"
            ],
            "compression_gzip": false,
            "use_logs_stream": false,
            "data_stream.dataset": "hostnamemappings",
            "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
            "exclude_files": [
              "\\.gz$"
            ],
            "include_files": [],
            "processors": "- decode_csv_fields:\n    fields:\n      message: decoded.csv\n    separator: \",\"\n    ignore_missing: false\n    overwrite_keys: true\n    trim_leading_space: true\n    fail_on_error: true\n\n- extract_array:\n    field: decoded.csv\n    mappings:\n      so.ip_address: '0'\n      so.description: '1'\n\n- script:\n    lang: javascript\n    source: >\n      function process(event) {\n          var ip = event.Get('so.ip_address');\n          var validIpRegex = /^((25[0-5]|2[0-4]\\d|1\\d{2}|[1-9]?\\d)\\.){3}(25[0-5]|2[0-4]\\d|1\\d{2}|[1-9]?\\d)$/\n          if (!validIpRegex.test(ip)) {\n              event.Cancel();\n          }\n      }\n- fingerprint:\n    fields: [\"so.ip_address\"]\n    target_field: \"@metadata._id\"\n",
            "tags": [
              "so-ip-mappings"
            ],
-            "processors": "- decode_csv_fields:\n    fields:\n      message: decoded.csv\n    separator: \",\"\n    ignore_missing: false\n    overwrite_keys: true\n    trim_leading_space: true\n    fail_on_error: true\n\n- extract_array:\n    field: decoded.csv\n    mappings:\n      so.ip_address: '0'\n      so.description: '1'\n\n- script:\n    lang: javascript\n    source: >\n      function process(event) {\n          var ip = event.Get('so.ip_address');\n          var validIpRegex = /^((25[0-5]|2[0-4]\\d|1\\d{2}|[1-9]?\\d)\\.){3}(25[0-5]|2[0-4]\\d|1\\d{2}|[1-9]?\\d)$/\n          if (!validIpRegex.test(ip)) {\n              event.Cancel();\n          }\n      }\n- fingerprint:\n    fields: [\"so.ip_address\"]\n    target_field: \"@metadata._id\"\n",
+            "recursive_glob": true,
-            "custom": ""
+            "clean_inactive": -1,
            "harvester_limit": 0,
            "fingerprint": false,
            "fingerprint_offset": 0,
            "file_identity_native": true,
            "exclude_lines": [],
            "include_lines": [],
            "delete_enabled": false
          }
        }
      }
@@ -31,5 +49,3 @@
  },
  "force": true
 }
@@ -1,30 +1,50 @@
 {
  "package": {
-    "name": "log",
+    "name": "filestream",
    "version": ""
  },
  "name": "soc-auth-sync-logs",
  "namespace": "so",
  "description": "Security Onion - Elastic Auth Sync - Logs",
  "policy_id": "so-grid-nodes_general",
  "policy_ids": [
    "so-grid-nodes_general"
  ],
  "vars": {},
  "inputs": {
-    "logs-logfile": {
+    "filestream-filestream": {
      "enabled": true,
      "streams": {
-        "log.logs": {
+        "filestream.filestream": {
          "enabled": true,
          "vars": {
            "paths": [
              "/opt/so/log/soc/sync.log"
            ],
            "compression_gzip": false,
            "use_logs_stream": false,
            "data_stream.dataset": "soc",
-            "tags": ["so-soc"],
+            "pipeline": "common",
            "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
            "exclude_files": [
              "\\.gz$"
            ],
            "include_files": [],
            "processors": "- dissect:\n    tokenizer: \"%{event.action}\"\n    field: \"message\"\n    target_prefix: \"\"\n- add_fields:\n    target: event\n    fields:\n      category: host\n      module: soc\n      dataset_temp: auth_sync",
-            "custom": "pipeline: common"
+            "tags": [],
            "recursive_glob": true,
            "clean_inactive": -1,
            "harvester_limit": 0,
            "fingerprint": false,
            "fingerprint_offset": 0,
            "file_identity_native": true,
            "exclude_lines": [],
            "include_lines": [],
            "delete_enabled": false
          }
        }
      }
    }
  },
  "force": true
-}
+}
@@ -1,35 +1,54 @@
 {
  "policy_id": "so-grid-nodes_general",
  "package": {
-    "name": "log",
+    "name": "filestream",
    "version": ""
  },
  "name": "soc-detections-logs",
  "description": "Security Onion Console - Detections Logs",
  "namespace": "so",
  "description": "Security Onion Console - Detections Logs",
  "policy_id": "so-grid-nodes_general",
  "policy_ids": [
    "so-grid-nodes_general"
  ],
  "vars": {},
  "inputs": {
-    "logs-logfile": {
+    "filestream-filestream": {
      "enabled": true,
      "streams": {
-        "log.logs": {
+        "filestream.filestream": {
          "enabled": true,
          "vars": {
            "paths": [
              "/opt/so/log/soc/detections_runtime-status_sigma.log",
              "/opt/so/log/soc/detections_runtime-status_yara.log"
            ],
-            "exclude_files": [],
+            "compression_gzip": false,
-            "ignore_older": "72h",
+            "use_logs_stream": false,
            "data_stream.dataset": "soc",
            "pipeline": "common",
            "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
            "exclude_files": [
              "\\.gz$"
            ],
            "include_files": [],
            "processors": "- decode_json_fields:\n    fields: [\"message\"]\n    target: \"soc\"\n    process_array: true\n    max_depth: 2\n    add_error_key: true      \n- add_fields:\n    target: event\n    fields:\n      category: host\n      module: soc\n      dataset_temp: detections\n- rename:\n    fields:\n      - from: \"soc.fields.sourceIp\"\n        to: \"source.ip\"\n      - from: \"soc.fields.status\"\n        to: \"http.response.status_code\"\n      - from: \"soc.fields.method\"\n        to: \"http.request.method\"\n      - from: \"soc.fields.path\"\n        to: \"url.path\"\n      - from: \"soc.message\"\n        to: \"event.action\"\n      - from: \"soc.level\"\n        to: \"log.level\"\n    ignore_missing: true",
            "tags": [
              "so-soc"
            ],
-            "processors": "- decode_json_fields:\n    fields: [\"message\"]\n    target: \"soc\"\n    process_array: true\n    max_depth: 2\n    add_error_key: true      \n- add_fields:\n    target: event\n    fields:\n      category: host\n      module: soc\n      dataset_temp: detections\n- rename:\n    fields:\n      - from: \"soc.fields.sourceIp\"\n        to: \"source.ip\"\n      - from: \"soc.fields.status\"\n        to: \"http.response.status_code\"\n      - from: \"soc.fields.method\"\n        to: \"http.request.method\"\n      - from: \"soc.fields.path\"\n        to: \"url.path\"\n      - from: \"soc.message\"\n        to: \"event.action\"\n      - from: \"soc.level\"\n        to: \"log.level\"\n    ignore_missing: true",
+            "recursive_glob": true,
-            "custom": "pipeline: common"
+            "ignore_older": "72h",
            "clean_inactive": -1,
            "harvester_limit": 0,
            "fingerprint": false,
            "fingerprint_offset": 0,
            "file_identity_native": true,
            "exclude_lines": [],
            "include_lines": [],
            "delete_enabled": false
          }
        }
      }
    }
  },
  "force": true
-}
+}
@@ -1,30 +1,52 @@
 {
  "package": {
-    "name": "log",
+    "name": "filestream",
    "version": ""
  },
  "name": "soc-salt-relay-logs",
  "namespace": "so",
  "description": "Security Onion - Salt Relay - Logs",
  "policy_id": "so-grid-nodes_general",
  "policy_ids": [
    "so-grid-nodes_general"
  ],
  "vars": {},
  "inputs": {
-    "logs-logfile": {
+    "filestream-filestream": {
      "enabled": true,
      "streams": {
-        "log.logs": {
+        "filestream.filestream": {
          "enabled": true,
          "vars": {
            "paths": [
              "/opt/so/log/soc/salt-relay.log"
            ],
            "compression_gzip": false,
            "use_logs_stream": false,
            "data_stream.dataset": "soc",
-            "tags": ["so-soc"],
+            "pipeline": "common",
            "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
            "exclude_files": [
              "\\.gz$"
            ],
            "include_files": [],
            "processors": "- dissect:\n    tokenizer: \"%{soc.ts} | %{event.action}\"\n    field: \"message\"\n    target_prefix: \"\"\n- add_fields:\n    target: event\n    fields:\n      category: host\n      module: soc\n      dataset_temp: salt_relay",
-            "custom": "pipeline: common"
+            "tags": [
              "so-soc"
            ],
            "recursive_glob": true,
            "clean_inactive": -1,
            "harvester_limit": 0,
            "fingerprint": false,
            "fingerprint_offset": 0,
            "file_identity_native": true,
            "exclude_lines": [],
            "include_lines": [],
            "delete_enabled": false
          }
        }
      }
    }
  },
  "force": true
-}
+}
@@ -1,30 +1,50 @@
 {
  "package": {
-    "name": "log",
+    "name": "filestream",
    "version": ""
  },
  "name": "soc-sensoroni-logs",
  "namespace": "so",
  "description": "Security Onion - Sensoroni - Logs",
  "policy_id": "so-grid-nodes_general",
  "policy_ids": [
    "so-grid-nodes_general"
  ],
  "vars": {},
  "inputs": {
-    "logs-logfile": {
+    "filestream-filestream": {
      "enabled": true,
      "streams": {
-        "log.logs": {
+        "filestream.filestream": {
          "enabled": true,
          "vars": {
            "paths": [
              "/opt/so/log/sensoroni/sensoroni.log"
            ],
            "compression_gzip": false,
            "use_logs_stream": false,
            "data_stream.dataset": "soc",
-            "tags": [],
+            "pipeline": "common",
            "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
            "exclude_files": [
              "\\.gz$"
            ],
            "include_files": [],
            "processors": "- decode_json_fields:\n    fields: [\"message\"]\n    target: \"sensoroni\"\n    process_array: true\n    max_depth: 2\n    add_error_key: true      \n- add_fields:\n    target: event\n    fields:\n      category: host\n      module: soc\n      dataset_temp: sensoroni\n- rename:\n    fields:\n      - from: \"sensoroni.fields.sourceIp\"\n        to: \"source.ip\"\n      - from: \"sensoroni.fields.status\"\n        to: \"http.response.status_code\"\n      - from: \"sensoroni.fields.method\"\n        to: \"http.request.method\"\n      - from: \"sensoroni.fields.path\"\n        to: \"url.path\"\n      - from: \"sensoroni.message\"\n        to: \"event.action\"\n      - from: \"sensoroni.level\"\n        to: \"log.level\"\n    ignore_missing: true",
-            "custom": "pipeline: common"
+            "tags": [],
            "recursive_glob": true,
            "clean_inactive": -1,
            "harvester_limit": 0,
            "fingerprint": false,
            "fingerprint_offset": 0,
            "file_identity_native": true,
            "exclude_lines": [],
            "include_lines": [],
            "delete_enabled": false
          }
        }
      }
    }
  },
-  "force": true
+"force": true
-}
+}
@@ -1,30 +1,52 @@
 {
  "package": {
-    "name": "log",
+    "name": "filestream",
    "version": ""
  },
  "name": "soc-server-logs",
  "namespace": "so",
  "description": "Security Onion Console Logs",
  "policy_id": "so-grid-nodes_general",
  "policy_ids": [
    "so-grid-nodes_general"
  ],
  "vars": {},
  "inputs": {
-    "logs-logfile": {
+    "filestream-filestream": {
      "enabled": true,
      "streams": {
-        "log.logs": {
+        "filestream.filestream": {
          "enabled": true,
          "vars": {
            "paths": [
              "/opt/so/log/soc/sensoroni-server.log"
            ],
            "compression_gzip": false,
            "use_logs_stream": false,
            "data_stream.dataset": "soc",
-            "tags": ["so-soc"],
+            "pipeline": "common",
            "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
            "exclude_files": [
              "\\.gz$"
            ],
            "include_files": [],
            "processors": "- decode_json_fields:\n    fields: [\"message\"]\n    target: \"soc\"\n    process_array: true\n    max_depth: 2\n    add_error_key: true      \n- add_fields:\n    target: event\n    fields:\n      category: host\n      module: soc\n      dataset_temp: server\n- rename:\n    fields:\n      - from: \"soc.fields.sourceIp\"\n        to: \"source.ip\"\n      - from: \"soc.fields.status\"\n        to: \"http.response.status_code\"\n      - from: \"soc.fields.method\"\n        to: \"http.request.method\"\n      - from: \"soc.fields.path\"\n        to: \"url.path\"\n      - from: \"soc.message\"\n        to: \"event.action\"\n      - from: \"soc.level\"\n        to: \"log.level\"\n    ignore_missing: true",
-            "custom": "pipeline: common"
+            "tags": [
              "so-soc"
            ],
            "recursive_glob": true,
            "clean_inactive": -1,
            "harvester_limit": 0,
            "fingerprint": false,
            "fingerprint_offset": 0,
            "file_identity_native": true,
            "exclude_lines": [],
            "include_lines": [],
            "delete_enabled": false
          }
        }
      }
    }
  },
  "force": true
-}
+}
@@ -1,30 +1,50 @@
 {
  "package": {
-    "name": "log",
+    "name": "filestream",
    "version": ""
  },
  "name": "strelka-logs",
  "namespace": "so",
-  "description": "Strelka logs",
+  "description": "Strelka Logs",
  "policy_id": "so-grid-nodes_general",
  "policy_ids": [
    "so-grid-nodes_general"
  ],
  "vars": {},
  "inputs": {
-    "logs-logfile": {
+    "filestream-filestream": {
      "enabled": true,
      "streams": {
-        "log.logs": {
+        "filestream.filestream": {
          "enabled": true,
          "vars": {
            "paths": [
              "/nsm/strelka/log/strelka.log"
            ],
            "compression_gzip": false,
            "use_logs_stream": false,
            "data_stream.dataset": "strelka",
-            "tags": [],
+            "pipeline": "strelka.file",
            "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
            "exclude_files": [
              "\\.gz$"
            ],
            "include_files": [],
            "processors": "- add_fields:\n    target: event\n    fields:\n      category: file\n      module: strelka",
-            "custom": "pipeline: strelka.file"
+            "tags": [],
            "recursive_glob": true,
            "clean_inactive": -1,
            "harvester_limit": 0,
            "fingerprint": false,
            "fingerprint_offset": 0,
            "file_identity_native": true,
            "exclude_lines": [],
            "include_lines": [],
            "delete_enabled": false
          }
        }
      }
    }
  },
  "force": true
-}
+}
@@ -1,26 +1,46 @@
 {
  "package": {
-    "name": "log",
+    "name": "filestream",
    "version": ""
  },
  "name": "suricata-logs",
  "namespace": "so",
  "description": "Suricata integration",
  "policy_id": "so-grid-nodes_general",
  "policy_ids": [
    "so-grid-nodes_general"
  ],
  "vars": {},
  "inputs": {
-    "logs-logfile": {
+    "filestream-filestream": {
      "enabled": true,
      "streams": {
-        "log.logs": {
+        "filestream.filestream": {
          "enabled": true,
          "vars": {
            "paths": [
              "/nsm/suricata/eve*.json"
            ],
            "compression_gzip": false,
            "use_logs_stream": false,
            "data_stream.dataset": "suricata",
-            "tags": [],
+            "pipeline": "suricata.common",
            "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
            "exclude_files": [
              "\\.gz$"
            ],
            "include_files": [],
            "processors": "- add_fields:\n    target: event\n    fields:\n      category: network\n      module: suricata",
-            "custom": "pipeline: suricata.common"
+            "tags": [],
            "recursive_glob": true,
            "clean_inactive": -1,
            "harvester_limit": 0,
            "fingerprint": false,
            "fingerprint_offset": 0,
            "file_identity_native": true,
            "exclude_lines": [],
            "include_lines": [],
            "delete_enabled": false
          }
        }
      }
@@ -0,0 +1,107 @@
 {
  "package": {
    "name": "elasticsearch",
    "version": ""
  },
  "name": "elasticsearch-grid-nodes_heavy",
  "namespace": "default",
  "description": "Elasticsearch Logs",
  "policy_id": "so-grid-nodes_heavy",
  "inputs": {
    "elasticsearch-logfile": {
      "enabled": true,
      "streams": {
        "elasticsearch.audit": {
          "enabled": false,
          "vars": {
            "paths": [
              "/var/log/elasticsearch/*_audit.json"
            ]
          }
        },
        "elasticsearch.deprecation": {
          "enabled": false,
          "vars": {
            "paths": [
              "/var/log/elasticsearch/*_deprecation.json"
            ]
          }
        },
        "elasticsearch.gc": {
          "enabled": false,
          "vars": {
            "paths": [
              "/var/log/elasticsearch/gc.log.[0-9]*",
              "/var/log/elasticsearch/gc.log"
            ]
          }
        },
        "elasticsearch.server": {
          "enabled": true,
          "vars": {
            "paths": [
              "/opt/so/log/elasticsearch/*.json"
            ]
          }
        },
        "elasticsearch.slowlog": {
          "enabled": false,
          "vars": {
            "paths": [
              "/var/log/elasticsearch/*_index_search_slowlog.json",
              "/var/log/elasticsearch/*_index_indexing_slowlog.json"
            ]
          }
        }
      }
    },
    "elasticsearch-elasticsearch/metrics": {
      "enabled": false,
      "vars": {
        "hosts": [
          "http://localhost:9200"
        ],
        "scope": "node"
      },
      "streams": {
        "elasticsearch.stack_monitoring.ccr": {
          "enabled": false
        },
        "elasticsearch.stack_monitoring.cluster_stats": {
          "enabled": false
        },
        "elasticsearch.stack_monitoring.enrich": {
          "enabled": false
        },
        "elasticsearch.stack_monitoring.index": {
          "enabled": false
        },
        "elasticsearch.stack_monitoring.index_recovery": {
          "enabled": false,
          "vars": {
            "active.only": true
          }
        },
        "elasticsearch.stack_monitoring.index_summary": {
          "enabled": false
        },
        "elasticsearch.stack_monitoring.ml_job": {
          "enabled": false
        },
        "elasticsearch.stack_monitoring.node": {
          "enabled": false
        },
        "elasticsearch.stack_monitoring.node_stats": {
          "enabled": false
        },
        "elasticsearch.stack_monitoring.pending_tasks": {
          "enabled": false
        },
        "elasticsearch.stack_monitoring.shard": {
          "enabled": false
        }
      }
    }
  },
  "force": true
 }
@@ -0,0 +1,123 @@
 {# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
  or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use
  this file except in compliance with the Elastic License 2.0. #}
 {% import_json '/opt/so/state/esfleet_input_package_components.json' as ADDON_INPUT_PACKAGE_COMPONENTS %}
 {% import_json '/opt/so/state/esfleet_component_templates.json' as INSTALLED_COMPONENT_TEMPLATES %}
 {% import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %}
 {% set CORE_ESFLEET_PACKAGES = ELASTICFLEETDEFAULTS.get('elasticfleet', {}).get('packages', {}) %}
 {% set ADDON_INPUT_INTEGRATION_DEFAULTS = {} %}
 {% set DEBUG_STUFF = {} %}
 {% for pkg in ADDON_INPUT_PACKAGE_COMPONENTS %}
 {%   if pkg.name in CORE_ESFLEET_PACKAGES %}
 {#     skip core input packages #}
 {%   elif pkg.name not in CORE_ESFLEET_PACKAGES %}
 {#     generate defaults for each input package #}
 {%     if pkg.dataStreams is defined and pkg.dataStreams is not none and pkg.dataStreams | length > 0 %}
 {%       for pattern in pkg.dataStreams %}
 {#         in ES 9.3.2 'input' type integrations no longer create default component templates and instead they wait for user input during 'integration' setup (fleet ui config)
             title: generic is an artifact of that and is not in use #}
 {%         if pattern.title == "generic" %}
 {%           continue %}
 {%         endif %}
 {%         if "metrics-" in pattern.name %}
 {%           set integration_type = "metrics-" %}
 {%         elif "logs-" in pattern.name %}
 {%           set integration_type = "logs-" %}
 {%         else %}
 {%           set integration_type = "" %}
 {%         endif %}
 {#         on input integrations the component name is user defined at the time it is added to an agent policy #}
 {%         set component_name = pattern.title %}
 {%         set index_pattern = pattern.name %}
 {#           component_name_x maintains the functionality of merging local pillar changes with generated 'defaults' via SOC UI #}
 {%           set component_name_x = component_name.replace(".","_x_") %}
 {#           pillar overrides/merge expects the key names to follow the naming in elasticsearch/defaults.yaml eg. so-logs-1password_x_item_usages . The _x_ is replaced later on in elasticsearch/template.map.jinja #}
 {%           set integration_key = "so-" ~ integration_type ~ pkg.name + '_x_' ~ component_name_x %}
 {#           Default integration settings #}
 {%           set integration_defaults = {
               "index_sorting": false,
               "index_template": {
                 "composed_of": [integration_type ~ component_name ~ "@package", integration_type ~ component_name ~ "@custom", "so-fleet_integrations.ip_mappings-1", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"],
                 "data_stream": {
                   "allow_custom_routing": false,
                   "hidden": false
                 },
                 "ignore_missing_component_templates": [integration_type ~ component_name ~ "@custom"],
                 "index_patterns": [index_pattern],
                 "priority": 501,
                 "template": {
                   "settings": {
                     "index": {
                       "lifecycle": {"name": "so-" ~ integration_type ~ component_name ~ "-logs"},
                         "number_of_replicas": 0
                     }
                   }
                 }
               },
               "policy": {
                 "phases": {
                   "cold": {
                     "actions": {
                        "allocate":{
                          "number_of_replicas": ""
                        },
                       "set_priority": {"priority": 0}
                     },
                     "min_age": "60d"
                   },
                   "delete": {
                     "actions": {
                       "delete": {}
                     },
                     "min_age": "365d"
                   },
                   "hot": {
                     "actions": {
                       "rollover": {
                         "max_age": "30d",
                         "max_primary_shard_size": "50gb"
                       },
                       "forcemerge":{
                         "max_num_segments": ""
                       },
                       "shrink":{
                         "max_primary_shard_size": "",
                         "method": "COUNT",
                         "number_of_shards": ""
                       },
                       "set_priority": {"priority": 100}
                      },
                      "min_age": "0ms"
                   },
                   "warm": {
                     "actions": {
                        "allocate": {
                          "number_of_replicas": ""
                        },
                        "forcemerge": {
                          "max_num_segments": ""
                        },
                        "shrink":{
                          "max_primary_shard_size": "",
                          "method": "COUNT",
                          "number_of_shards": ""
                        },
                       "set_priority": {"priority": 50}
                     },
                     "min_age": "30d"
                   }
                 }
               }
             } %}
 {%         do ADDON_INPUT_INTEGRATION_DEFAULTS.update({integration_key: integration_defaults}) %}
 {%         do DEBUG_STUFF.update({integration_key: "Generating defaults for "+ pkg.name })%}
 {%       endfor %}
 {%     endif %}
 {%   endif %}
 {% endfor %}
@@ -2,26 +2,32 @@
 # or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use
 # this file except in compliance with the Elastic License 2.0.
-{%- set GRIDNODETOKENGENERAL = salt['pillar.get']('global:fleet_grid_enrollment_token_general') -%}
+{% set GRIDNODETOKEN = salt['pillar.get']('global:fleet_grid_enrollment_token_general') -%}
-{%- set GRIDNODETOKENHEAVY = salt['pillar.get']('global:fleet_grid_enrollment_token_heavy') -%}
+{% if grains.role == 'so-heavynode' %}
 {%   set GRIDNODETOKEN = salt['pillar.get']('global:fleet_grid_enrollment_token_heavy') -%}
 {% endif %}
 {% set AGENT_STATUS = salt['service.available']('elastic-agent') %}
-{% if not AGENT_STATUS  %}
+{% set AGENT_EXISTS = salt['file.file_exists']('/opt/Elastic/Agent/elastic-agent') %}
-{% if grains.role not in ['so-heavynode'] %}
+{% if not AGENT_STATUS or not AGENT_EXISTS %}
 run_installer:
  cmd.script:
    - name: salt://elasticfleet/files/so_agent-installers/so-elastic-agent_linux_amd64
    - cwd: /opt/so
    - args: -token={{ GRIDNODETOKENGENERAL }}
    - retry: True
 {% else %} 
 run_installer:
  cmd.script:
    - name: salt://elasticfleet/files/so_agent-installers/so-elastic-agent_linux_amd64
    - cwd: /opt/so
    - args: -token={{ GRIDNODETOKENHEAVY }}
    - retry: True
 {% endif %}  
 pull_agent_installer:
  file.managed:
    - name: /opt/so/so-elastic-agent_linux_amd64
    - source: salt://elasticfleet/files/so_agent-installers/so-elastic-agent_linux_amd64
    - mode: 755
    - makedirs: True
 run_installer:
  cmd.run:
    - name: ./so-elastic-agent_linux_amd64 -token={{ GRIDNODETOKEN }} -force
    - cwd: /opt/so
    - retry:
        attempts: 3
        interval: 20
 cleanup_agent_installer:
  file.absent:
    - name: /opt/so/so-elastic-agent_linux_amd64
 {% endif %}
@@ -21,6 +21,7 @@
    'azure_application_insights.app_state': 'azure.app_state',
    'azure_billing.billing': 'azure.billing',
    'azure_functions.metrics': 'azure.function',
    'azure_ai_foundry.metrics': 'azure.ai_foundry',
    'azure_metrics.compute_vm_scaleset': 'azure.compute_vm_scaleset',
    'azure_metrics.compute_vm': 'azure.compute_vm',
    'azure_metrics.container_instance': 'azure.container_instance',
@@ -58,8 +59,8 @@
 {#     skip core integrations #}
 {%   elif pkg.name not in CORE_ESFLEET_PACKAGES %}
 {#     generate defaults for each integration #}
-{%     if pkg.es_index_patterns is defined and pkg.es_index_patterns is not none %}
+{%     if pkg.dataStreams is defined and pkg.dataStreams is not none and pkg.dataStreams | length > 0 %}
-{%       for pattern in pkg.es_index_patterns %}
+{%       for pattern in pkg.dataStreams %}
 {%         if "metrics-" in pattern.name %}
 {%           set integration_type = "metrics-" %}
 {%         elif "logs-" in pattern.name %}
@@ -74,44 +75,27 @@
 {%         if component_name in WEIRD_INTEGRATIONS %}
 {%           set component_name = WEIRD_INTEGRATIONS[component_name] %}
 {%         endif %}
 {#         create duplicate of component_name, so we can split generics from @custom component templates in the index template below and overwrite the default @package when needed
           eg. having to replace unifiedlogs.generic@package with filestream.generic@package, but keep the ability to customize unifiedlogs.generic@custom and its ILM policy #}
 {%         set custom_component_name = component_name %}
 {#         duplicate integration_type to assist with sometimes needing to overwrite component templates with 'logs-filestream.generic@package' (there is no metrics-filestream.generic@package) #}
 {%         set generic_integration_type = integration_type %}
 {#           component_name_x maintains the functionality of merging local pillar changes with generated 'defaults' via SOC UI #}
 {%           set component_name_x = component_name.replace(".","_x_") %}
 {#           pillar overrides/merge expects the key names to follow the naming in elasticsearch/defaults.yaml eg. so-logs-1password_x_item_usages . The _x_ is replaced later on in elasticsearch/template.map.jinja #}
 {%           set integration_key = "so-" ~ integration_type ~ component_name_x %}
 {#         if its a .generic template make sure that a .generic@package for the integration exists. Else default to logs-filestream.generic@package #}
 {%         if ".generic" in component_name and integration_type ~ component_name ~ "@package" not in INSTALLED_COMPONENT_TEMPLATES %}
 {#           these generic templates by default are directed to index_pattern of 'logs-generic-*', overwrite that here to point to eg gcp_pubsub.generic-* #}
 {%           set index_pattern = integration_type ~ component_name ~ "-*" %}
 {#           includes use of .generic component template, but it doesn't exist in installed component templates. Redirect it to filestream.generic@package #}
 {%           set component_name = "filestream.generic" %}
 {%           set generic_integration_type = "logs-" %}
 {%         endif %}
 {#           Default integration settings #}
 {%           set integration_defaults = {
               "index_sorting": false,
               "index_template": {
-                 "composed_of": [generic_integration_type ~ component_name ~ "@package", integration_type ~ custom_component_name ~ "@custom", "so-fleet_integrations.ip_mappings-1", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"],
+                 "composed_of": [integration_type ~ component_name ~ "@package", integration_type ~ component_name ~ "@custom", "so-fleet_integrations.ip_mappings-1", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"],
                 "data_stream": {
                   "allow_custom_routing": false,
                   "hidden": false
                 },
-                 "ignore_missing_component_templates": [integration_type ~ custom_component_name ~ "@custom"],
+                 "ignore_missing_component_templates": [integration_type ~ component_name ~ "@custom"],
                 "index_patterns": [index_pattern],
                 "priority": 501,
                 "template": {
                   "settings": {
                     "index": {
-                       "lifecycle": {"name": "so-" ~ integration_type ~ custom_component_name ~ "-logs"},
+                       "lifecycle": {"name": "so-" ~ integration_type ~ component_name ~ "-logs"},
                         "number_of_replicas": 0
                     }
                   }
@@ -121,6 +105,9 @@
                 "phases": {
                   "cold": {
                     "actions": {
                        "allocate":{
                          "number_of_replicas": ""
                        },
                       "set_priority": {"priority": 0}
                     },
                     "min_age": "60d"
@@ -137,12 +124,31 @@
                         "max_age": "30d",
                         "max_primary_shard_size": "50gb"
                       },
                       "forcemerge":{
                         "max_num_segments": ""
                       },
                       "shrink":{
                         "max_primary_shard_size": "",
                         "method": "COUNT",
                         "number_of_shards": ""
                       },
                       "set_priority": {"priority": 100}
                      },
                      "min_age": "0ms"
                   },
                   "warm": {
                     "actions": {
                        "allocate": {
                          "number_of_replicas": ""
                        },
                        "forcemerge": {
                          "max_num_segments": ""
                        },
                        "shrink":{
                          "max_primary_shard_size": "",
                          "method": "COUNT",
                          "number_of_shards": ""
                        },
                       "set_priority": {"priority": 50}
                     },
                     "min_age": "30d"
@@ -1,14 +1,15 @@
 elasticfleet:
  enabled:
    description: Enables or disables the Elastic Fleet process. This process is critical for managing Elastic Agents.
    forcedType: bool
    advanced: True
-    helpLink: elastic-fleet.html
+    helpLink: elastic-fleet
  enable_manager_output:
    description: Setting this option to False should only be considered if there is at least one receiver node in the grid. If True, Elastic Agent will send events to the manager and receivers. If False, events will only be send to the receivers.
    advanced: True
    global: True
    forcedType: bool
-    helpLink: elastic-fleet.html
+    helpLink: elastic-fleet
  files:
    soc:
      elastic-defend-disabled-filters__yaml:
@@ -17,7 +18,7 @@ elasticfleet:
        syntax: yaml
        file: True
        global: True
-        helpLink: elastic-fleet.html
+        helpLink: elastic-fleet
        advanced: True
      elastic-defend-custom-filters__yaml:
        title: Custom Elastic Defend filters
@@ -25,59 +26,101 @@ elasticfleet:
        syntax: yaml
        file: True
        global: True
-        helpLink: elastic-fleet.html
+        helpLink: elastic-fleet
        advanced: True
  logging:
    zeek:
      excluded:
        description: This is a list of Zeek logs that are excluded from being shipped through the data processing pipeline. If you remove a log from this list, Elastic Agent will attempt to process it. If an ingest node pipeline is not available to process the logs, you may experience errors.
        forcedType: "[]string"
-        helpLink: zeek.html
+        helpLink: zeek
  config:
    defend_filters:
      enable_auto_configuration:
        description: Enable auto-configuration and management of the Elastic Defend Exclusion filters.
        forcedType: bool
        global: True
-        helpLink: elastic-fleet.html
+        helpLink: elastic-fleet
        advanced: True
    subscription_integrations:
      description: Enable the installation of integrations that require an Elastic license.
      global: True
      forcedType: bool
-      helpLink: elastic-fleet.html
+      helpLink: elastic-fleet
    auto_upgrade_integrations:
      description: Enables or disables automatically upgrading Elastic Agent integrations.
      global: True
      forcedType: bool
-      helpLink: elastic-fleet.html
+      helpLink: elastic-fleet
    outputs:
      logstash:
        bulk_max_size:
          description: The maximum number of events to bulk in a single Logstash request.
          global: True
          forcedType: int
          advanced: True
          helpLink: elastic-fleet
        worker:
          description: The number of workers per configured host publishing events.
          global: True
          forcedType: int
          advanced: true
          helpLink: elastic-fleet
        queue_mem_events:
          title: queued events
          description: The number of events the queue can store. This value should be evenly divisible by the smaller of 'bulk_max_size' to avoid sending partial batches to the output.
          global: True
          forcedType: int
          advanced: True
          helpLink: elastic-fleet
        timeout:
          description: The number of seconds to wait for responses from the Logstash server before timing out. Eg 30s
          regex: ^[0-9]+s$
          advanced: True
          global: True
          helpLink: elastic-fleet
        loadbalance:
          description: If true and multiple Logstash hosts are configured, the output plugin load balances published events onto all Logstash hosts. If false, the output plugin sends all events to one host (determined at random) and switches to another host if the selected one becomes unresponsive.
          forcedType: bool
          advanced: True
          global: True
          helpLink: elastic-fleet
        compression_level:
          description: The gzip compression level. The compression level must be in the range of 1 (best speed) to 9 (best compression).
          regex: ^[1-9]$
          forcedType: int
          advanced: True
          global: True
          helpLink: elastic-fleet
    server:
      custom_fqdn:
        description: Custom FQDN for Agents to connect to. One per line.
        global: True
-        helpLink: elastic-fleet.html
+        helpLink: elastic-fleet
        advanced: True
        forcedType: "[]string"
      enable_auto_configuration:
        description: Enable auto-configuration of Logstash Outputs & Fleet Host URLs.
        forcedType: bool
        global: True
-        helpLink: elastic-fleet.html
+        helpLink: elastic-fleet
        advanced: True
      endpoints_enrollment:
        description: Endpoint enrollment key.
        global: True
-        helpLink: elastic-fleet.html
+        helpLink: elastic-fleet
        sensitive: True
        advanced: True
      es_token:
        description: Elastic auth token.
        global: True
-        helpLink: elastic-fleet.html
+        helpLink: elastic-fleet
        sensitive: True
        advanced: True
      grid_enrollment:
        description: Grid enrollment key.
        global: True
-        helpLink: elastic-fleet.html
+        helpLink: elastic-fleet
        sensitive: True
        advanced: True
  optional_integrations:
@@ -85,57 +128,57 @@ elasticfleet:
      enabled_nodes:
        description: Fleet nodes with the Sublime Platform integration enabled. Enter one per line.
        global: True
-        helpLink: elastic-fleet.html
+        helpLink: elastic-fleet
        advanced: True
        forcedType: "[]string"
      api_key:
        description: API key for Sublime Platform.
        global: True
-        helpLink: elastic-fleet.html
+        helpLink: elastic-fleet
        advanced: True
        forcedType: string
        sensitive: True
      base_url:
        description: Base URL for Sublime Platform.
        global: True
-        helpLink: elastic-fleet.html
+        helpLink: elastic-fleet
        advanced: True
        forcedType: string
      poll_interval:
        description: Poll interval for alerts from Sublime Platform.
        global: True
-        helpLink: elastic-fleet.html
+        helpLink: elastic-fleet
        advanced: True
        forcedType: string
      limit:
        description: The maximum number of message groups to return from Sublime Platform.
        global: True
-        helpLink: elastic-fleet.html
+        helpLink: elastic-fleet
        advanced: True
        forcedType: int
    kismet:
      base_url:
        description: Base URL for Kismet.
        global: True
-        helpLink: elastic-fleet.html
+        helpLink: elastic-fleet
        advanced: True
        forcedType: string
      poll_interval:
        description: Poll interval for wireless device data from Kismet. Integration is currently configured to return devices seen as active by any Kismet sensor within the last 10 minutes.
        global: True
-        helpLink: elastic-fleet.html
+        helpLink: elastic-fleet
        advanced: True
        forcedType: string
      api_key:
        description: API key for Kismet.
        global: True
-        helpLink: elastic-fleet.html
+        helpLink: elastic-fleet
        advanced: True
        forcedType: string
        sensitive: True
      enabled_nodes:
        description: Fleet nodes with the Kismet integration enabled. Enter one per line.
        global: True
-        helpLink: elastic-fleet.html
+        helpLink: elastic-fleet
        advanced: True
        forcedType: "[]string"
@@ -0,0 +1,186 @@
 # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
 # or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 {%   from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %}
 {%   from 'ca/map.jinja' import CA %}
 {% if GLOBALS.is_manager or GLOBALS.role in ['so-heavynode', 'so-fleet', 'so-receiver'] %}
 {% if grains['role'] not in [ 'so-heavynode', 'so-receiver'] %}
 # Start -- Elastic Fleet Host Cert
 etc_elasticfleet_key:
  x509.private_key_managed:
    - name: /etc/pki/elasticfleet-server.key
    - keysize: 4096
    - backup: True
    - new: True
    {% if salt['file.file_exists']('/etc/pki/elasticfleet-server.key') -%}
    - prereq:
      - x509: etc_elasticfleet_crt
    {%- endif %}
    - retry:
        attempts: 5
        interval: 30
 etc_elasticfleet_crt:
  x509.certificate_managed:
    - name: /etc/pki/elasticfleet-server.crt
    - ca_server: {{ CA.server }}
    - signing_policy: elasticfleet
    - private_key: /etc/pki/elasticfleet-server.key
    - CN: {{ GLOBALS.hostname }}
    - subjectAltName: DNS:{{ GLOBALS.hostname }},DNS:{{ GLOBALS.url_base }},IP:{{ GLOBALS.node_ip }}{% if ELASTICFLEETMERGED.config.server.custom_fqdn | length > 0 %},DNS:{{ ELASTICFLEETMERGED.config.server.custom_fqdn | join(',DNS:') }}{% endif %}
    - days_remaining: 7
    - days_valid: 820
    - backup: True
    - timeout: 30
    - retry:
        attempts: 5
        interval: 30
 efperms:
  file.managed:
    - replace: False
    - name: /etc/pki/elasticfleet-server.key
    - mode: 640
    - group: 939
 chownelasticfleetcrt:
  file.managed:
    - replace: False
    - name: /etc/pki/elasticfleet-server.crt
    - mode: 640
    - user: 947
    - group: 939
 chownelasticfleetkey:
  file.managed:
    - replace: False
    - name: /etc/pki/elasticfleet-server.key
    - mode: 640
    - user: 947
    - group: 939
 # End -- Elastic Fleet Host Cert
 {% endif %} # endif is for not including HeavyNodes & Receivers 
 # Start -- Elastic Fleet Client Cert for Agent (Mutual Auth with Logstash Output)
 etc_elasticfleet_agent_key:
  x509.private_key_managed:
    - name: /etc/pki/elasticfleet-agent.key
    - keysize: 4096
    - backup: True
    - new: True
    {% if salt['file.file_exists']('/etc/pki/elasticfleet-agent.key') -%}
    - prereq:
      - x509: etc_elasticfleet_agent_crt
    {%- endif %}
    - retry:
        attempts: 5
        interval: 30
 etc_elasticfleet_agent_crt:
  x509.certificate_managed:
    - name: /etc/pki/elasticfleet-agent.crt
    - ca_server: {{ CA.server }}
    - signing_policy: elasticfleet
    - private_key: /etc/pki/elasticfleet-agent.key
    - CN: {{ GLOBALS.hostname }}
    - days_remaining: 7
    - days_valid: 820
    - backup: True
    - timeout: 30
    - retry:
        attempts: 5
        interval: 30
  cmd.run:
    - name: "/usr/bin/openssl pkcs8 -in /etc/pki/elasticfleet-agent.key -topk8 -out /etc/pki/elasticfleet-agent.p8 -nocrypt"
    - onchanges:
      - x509: etc_elasticfleet_agent_key
 efagentperms:
  file.managed:
    - replace: False
    - name: /etc/pki/elasticfleet-agent.key
    - mode: 640
    - group: 939
 chownelasticfleetagentcrt:
  file.managed:
    - replace: False
    - name: /etc/pki/elasticfleet-agent.crt
    - mode: 640
    - user: 947
    - group: 939
 chownelasticfleetagentkey:
  file.managed:
    - replace: False
    - name: /etc/pki/elasticfleet-agent.key
    - mode: 640
    - user: 947
    - group: 939
 # End -- Elastic Fleet Client Cert for Agent (Mutual Auth with Logstash Output)
 {% endif %}
 {% if GLOBALS.role in ['so-manager', 'so-managerhype', 'so-managersearch', 'so-standalone'] %}
 elasticfleet_kafka_key:
  x509.private_key_managed:
    - name: /etc/pki/elasticfleet-kafka.key
    - keysize: 4096
    - backup: True
    - new: True
    {% if salt['file.file_exists']('/etc/pki/elasticfleet-kafka.key') -%}
    - prereq:
      - x509: elasticfleet_kafka_crt
    {%- endif %}
    - retry:
        attempts: 5
        interval: 30
 elasticfleet_kafka_crt:
  x509.certificate_managed:
    - name: /etc/pki/elasticfleet-kafka.crt
    - ca_server: {{ CA.server }}
    - signing_policy: kafka
    - private_key: /etc/pki/elasticfleet-kafka.key
    - CN: {{ GLOBALS.hostname }}
    - subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }}
    - days_remaining: 7
    - days_valid: 820
    - backup: True
    - timeout: 30
    - retry:
        attempts: 5
        interval: 30
 elasticfleet_kafka_cert_perms:
  file.managed:
    - replace: False
    - name: /etc/pki/elasticfleet-kafka.crt
    - mode: 640
    - user: 947
    - group: 939
 elasticfleet_kafka_key_perms:
  file.managed:
    - replace: False
    - name: /etc/pki/elasticfleet-kafka.key
    - mode: 640
    - user: 947
    - group: 939
 {% endif %}
 {% else %}
 {{sls}}_state_not_allowed:
  test.fail_without_changes:
    - name: {{sls}}_state_not_allowed
 {% endif %}
@@ -23,6 +23,13 @@ fi
 # Define a banner to separate sections
 banner="========================================================================="
 fleet_api() {
    local QUERYPATH=$1
    shift
    curl -sK /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/${QUERYPATH}" "$@" --retry 3 --retry-delay 10 --fail 2>/dev/null
 }
 elastic_fleet_integration_check() {
    AGENT_POLICY=$1
@@ -39,7 +46,9 @@ elastic_fleet_integration_create() {
    JSON_STRING=$1
-    curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
+    if ! fleet_api "package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -XPOST -d "$JSON_STRING"; then
        return 1
    fi
 }
@@ -56,7 +65,10 @@ elastic_fleet_integration_remove() {
                    '{"packagePolicyIds":[$INTEGRATIONID]}'
                    )
-    curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/package_policies/delete" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
+    if ! fleet_api "package_policies/delete" -XPOST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"; then
        echo "Error: Unable to delete '$NAME' from '$AGENT_POLICY'"
        return 1
    fi
 }
 elastic_fleet_integration_update() {
@@ -65,7 +77,9 @@ elastic_fleet_integration_update() {
    JSON_STRING=$2
-    curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/package_policies/$UPDATE_ID" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
+    if ! fleet_api "package_policies/$UPDATE_ID" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -XPUT -d "$JSON_STRING"; then
        return 1
    fi
 }
 elastic_fleet_integration_policy_upgrade() {
@@ -77,78 +91,107 @@ elastic_fleet_integration_policy_upgrade() {
                    '{"packagePolicyIds":[$INTEGRATIONID]}'
                    )
-    curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/package_policies/upgrade" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
+    if ! fleet_api "package_policies/upgrade" -XPOST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"; then
        return 1
    fi
 }
 elastic_fleet_package_version_check() {
    PACKAGE=$1
-    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/epm/packages/$PACKAGE" | jq -r '.item.version'
+
    if output=$(fleet_api "epm/packages/$PACKAGE"); then
        echo "$output" | jq -r '.item.version'
    else
        echo "Error: Failed to get current package version for '$PACKAGE'"
        return 1
    fi
 }
 elastic_fleet_package_latest_version_check() {
    PACKAGE=$1
-    if output=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/epm/packages/$PACKAGE" --fail); then
+    if output=$(fleet_api "epm/packages/$PACKAGE"); then
        if version=$(jq -e -r '.item.latestVersion' <<< $output); then
            echo "$version"
        fi
    else
-        echo "Error: Failed to get latest version for $PACKAGE"
+        echo "Error: Failed to get latest version for '$PACKAGE'"
        return 1
    fi
 }
 elastic_fleet_package_install() {
    PKG=$1
    VERSION=$2
-    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST -H 'kbn-xsrf: true' -H 'Content-Type: application/json'  -d '{"force":true}' "localhost:5601/api/fleet/epm/packages/$PKG/$VERSION"
+    if ! fleet_api "epm/packages/$PKG/$VERSION" -XPOST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{"force":true}'; then
        return 1
    fi
 }
 elastic_fleet_bulk_package_install() {
    BULK_PKG_LIST=$1
-    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d@$1 "localhost:5601/api/fleet/epm/packages/_bulk"
+    if ! fleet_api "epm/packages/_bulk" -XPOST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d@$BULK_PKG_LIST; then
-}
+        return 1
 elastic_fleet_package_is_installed() {
    PACKAGE=$1
    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET -H 'kbn-xsrf: true' "localhost:5601/api/fleet/epm/packages/$PACKAGE" | jq -r '.item.status'
 }
 elastic_fleet_installed_packages() {
    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET -H 'kbn-xsrf: true' -H 'Content-Type: application/json' "localhost:5601/api/fleet/epm/packages/installed?perPage=500"
 }
 elastic_fleet_agent_policy_ids() {
    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/agent_policies" | jq -r .items[].id
    if [ $? -ne 0 ]; then
        echo "Error: Failed to retrieve agent policies."
        exit 1
    fi
 }
-elastic_fleet_agent_policy_names() {
+elastic_fleet_get_package_list_by_type() {
-    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/agent_policies" | jq -r .items[].name
+    if ! output=$(fleet_api "epm/packages"); then
-    if [ $? -ne 0 ]; then
+        return 1
    else
        is_integration=$(jq '[.items[] | select(.type=="integration") | .name ]' <<< "$output")
        is_input=$(jq '[.items[] | select(.type=="input") | .name ]' <<< "$output")
        is_content=$(jq '[.items[] | select(.type=="content") | .name ]' <<< "$output")
        jq -n --argjson is_integration "${is_integration:-[]}" \
              --argjson is_input "${is_input:-[]}" \
              --argjson is_content "${is_content:-[]}" \
              '{"integration": $is_integration,"input": $is_input, "content": $is_content}'
    fi
 }
 elastic_fleet_installed_packages_components() {
    package_type=${1,,}
    if [[ "$package_type" != "integration" && "$package_type" != "input" && "$package_type" != "content" ]]; then
        echo "Error: Invalid package type ${package_type}. Valid types are 'integration', 'input', or 'content'."
        return 1
    fi
    packages_by_type=$(elastic_fleet_get_package_list_by_type)
    packages=$(jq --arg package_type "$package_type" '.[$package_type]' <<< "$packages_by_type")
    if ! output=$(fleet_api "epm/packages/installed?perPage=500"); then
        return 1
    else
        jq -c --argjson packages "$packages" '[.items[] | select(.name | IN($packages[])) | {name: .name, dataStreams: .dataStreams}]' <<< "$output"
    fi
 }
 elastic_fleet_agent_policy_ids() {
    if output=$(fleet_api "agent_policies"); then
        echo "$output" | jq -r .items[].id
    else
        echo "Error: Failed to retrieve agent policies."
-        exit 1
+        return 1
    fi
 }
 elastic_fleet_integration_policy_names() {
    AGENT_POLICY=$1
-    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/agent_policies/$AGENT_POLICY" | jq -r .item.package_policies[].name
+    if output=$(fleet_api "agent_policies/$AGENT_POLICY"); then
-    if [ $? -ne 0 ]; then
+        echo "$output" | jq -r .item.package_policies[].name
    else
        echo "Error: Failed to retrieve integrations for '$AGENT_POLICY'."
-        exit 1
+        return 1
    fi
 }
 elastic_fleet_integration_policy_package_name() {
    AGENT_POLICY=$1
    INTEGRATION=$2
-    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/agent_policies/$AGENT_POLICY" | jq -r --arg INTEGRATION "$INTEGRATION" '.item.package_policies[] | select(.name==$INTEGRATION)| .package.name'
+    if output=$(fleet_api "agent_policies/$AGENT_POLICY"); then
-    if [ $? -ne 0 ]; then
+        echo "$output" | jq -r --arg INTEGRATION "$INTEGRATION" '.item.package_policies[] | select(.name==$INTEGRATION)| .package.name'
    else
        echo "Error: Failed to retrieve package name for '$INTEGRATION' in '$AGENT_POLICY'."
-        exit 1
+        return 1
    fi
 }
@@ -156,32 +199,32 @@ elastic_fleet_integration_policy_package_version() {
    AGENT_POLICY=$1
    INTEGRATION=$2
-    if output=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -L -X GET "localhost:5601/api/fleet/agent_policies/$AGENT_POLICY" --fail); then
+    if output=$(fleet_api "agent_policies/$AGENT_POLICY"); then
-        if version=$(jq -e -r --arg INTEGRATION "$INTEGRATION" '.item.package_policies[] | select(.name==$INTEGRATION)| .package.version' <<< $output); then
+        if version=$(jq -e -r --arg INTEGRATION "$INTEGRATION" '.item.package_policies[] | select(.name==$INTEGRATION)| .package.version' <<< "$output"); then
            echo "$version"
        fi
    else
-        echo "Error: Failed to retrieve agent policy $AGENT_POLICY"
+        echo "Error: Failed to retrieve integration version for '$INTEGRATION' in policy '$AGENT_POLICY'"
-        exit 1
+        return 1
    fi
 }
 elastic_fleet_integration_id() {
    AGENT_POLICY=$1
    INTEGRATION=$2
-    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/agent_policies/$AGENT_POLICY" | jq -r --arg INTEGRATION "$INTEGRATION" '.item.package_policies[] | select(.name==$INTEGRATION)| .id'
+    if output=$(fleet_api "agent_policies/$AGENT_POLICY"); then
-    if [ $? -ne 0 ]; then
+        echo "$output" | jq -r --arg INTEGRATION "$INTEGRATION" '.item.package_policies[] | select(.name==$INTEGRATION)| .id'
    else
        echo "Error: Failed to retrieve integration ID for '$INTEGRATION' in '$AGENT_POLICY'."
-        exit 1
+        return 1
    fi
 }
 elastic_fleet_integration_policy_dryrun_upgrade() {
    INTEGRATION_ID=$1
-    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -H "Content-Type: application/json" -H 'kbn-xsrf: true' -L -X POST "localhost:5601/api/fleet/package_policies/upgrade/dryrun" -d "{\"packagePolicyIds\":[\"$INTEGRATION_ID\"]}"
+    if ! fleet_api "package_policies/upgrade/dryrun" -H "Content-Type: application/json" -H 'kbn-xsrf: true' -XPOST -d "{\"packagePolicyIds\":[\"$INTEGRATION_ID\"]}"; then
    if [ $? -ne 0 ]; then
        echo "Error: Failed to complete dry run for '$INTEGRATION_ID'."
-        exit 1
+        return 1
    fi
 }
@@ -190,25 +233,18 @@ elastic_fleet_policy_create() {
    NAME=$1
    DESC=$2
    FLEETSERVER=$3
-        TIMEOUT=$4
+    TIMEOUT=$4
    JSON_STRING=$( jq -n \
-                    --arg NAME "$NAME" \
+        --arg NAME "$NAME" \
-                    --arg DESC "$DESC" \
+        --arg DESC "$DESC" \
-                    --arg TIMEOUT $TIMEOUT \
+        --arg TIMEOUT $TIMEOUT \
-                    --arg FLEETSERVER "$FLEETSERVER" \
+        --arg FLEETSERVER "$FLEETSERVER" \
-                    '{"name": $NAME,"id":$NAME,"description":$DESC,"namespace":"default","monitoring_enabled":["logs"],"inactivity_timeout":$TIMEOUT,"has_fleet_server":$FLEETSERVER}'
+            '{"name": $NAME,"id":$NAME,"description":$DESC,"namespace":"default","monitoring_enabled":["logs"],"inactivity_timeout":$TIMEOUT,"has_fleet_server":$FLEETSERVER}'
-                    )
+        )
    # Create Fleet Policy
-    curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/agent_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
+    if ! fleet_api "agent_policies" -XPOST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"; then
        return 1
    fi
 }
 elastic_fleet_policy_update() {
    POLICYID=$1
    JSON_STRING=$2
    curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/agent_policies/$POLICYID" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
 }
@@ -8,6 +8,7 @@
 . /usr/sbin/so-elastic-fleet-common
 ERROR=false
 # Manage Elastic Defend Integration for Initial Endpoints Policy
 for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/elastic-defend/*.json
 do
@@ -15,9 +16,20 @@ do
  elastic_fleet_integration_check "endpoints-initial" "$INTEGRATION"
  if [ -n "$INTEGRATION_ID" ]; then
      printf "\n\nIntegration $NAME exists - Upgrading integration policy\n"
-      elastic_fleet_integration_policy_upgrade "$INTEGRATION_ID"
+      if ! elastic_fleet_integration_policy_upgrade "$INTEGRATION_ID"; then
        echo -e "\nFailed to upgrade integration policy for ${INTEGRATION##*/}"
        ERROR=true
        continue
      fi
  else
    printf "\n\nIntegration does not exist - Creating integration\n"
-    elastic_fleet_integration_create "@$INTEGRATION"
+    if ! elastic_fleet_integration_create "@$INTEGRATION"; then
        echo -e "\nFailed to create integration for ${INTEGRATION##*/}"
        ERROR=true
        continue
    fi
  fi
 done
 if [[ "$ERROR" == "true" ]]; then
    exit 1
 fi
@@ -25,5 +25,9 @@ for POLICYNAME in $POLICY; do
    .name = $name' /opt/so/conf/elastic-fleet/integrations/fleet-server/fleet-server.json)
    # Now update the integration policy using the modified JSON
-    elastic_fleet_integration_update "$INTEGRATION_ID" "$UPDATED_INTEGRATION_POLICY"
+    if ! elastic_fleet_integration_update "$INTEGRATION_ID" "$UPDATED_INTEGRATION_POLICY"; then
        # exit 1 on failure to update fleet integration policies, let salt handle retries
        echo "Failed to update $POLICYNAME.."
        exit 1
    fi
 done
--- a/Show More
+++ b/Show More