Merge pull request #8086 from Security-Onion-Solutions/dev

2.3.130

.github/.gitleaks.toml

@@ -0,0 +1,546 @@
+title = "gitleaks config"
+
+# Gitleaks rules are defined by regular expressions and entropy ranges.
+# Some secrets have unique signatures which make detecting those secrets easy.
+# Examples of those secrets would be GitLab Personal Access Tokens, AWS keys, and GitHub Access Tokens.
+# All these examples have defined prefixes like `glpat`, `AKIA`, `ghp_`, etc.
+#
+# Other secrets might just be a hash which means we need to write more complex rules to verify
+# that what we are matching is a secret.
+#
+# Here is an example of a semi-generic secret
+#
+#   discord_client_secret = "8dyfuiRyq=vVc3RRr_edRk-fK__JItpZ"
+#
+# We can write a regular expression to capture the variable name (identifier),
+# the assignment symbol (like '=' or ':='), and finally the actual secret.
+# The structure of a rule to match this example secret is below:
+#
+#                                                           Beginning string
+#                                                               quotation
+#                                                                   │            End string quotation
+#                                                                   │                      │
+#                                                                   ▼                      ▼
+#    (?i)(discord[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9=_\-]{32})['\"]
+#
+#                   ▲                              ▲                                ▲
+#                   │                              │                                │
+#                   │                              │                                │
+#              identifier                  assignment symbol
+#                                                                                Secret
+#
+[[rules]]
+id = "gitlab-pat"
+description = "GitLab Personal Access Token"
+regex = '''glpat-[0-9a-zA-Z\-\_]{20}'''
+
+[[rules]]
+id = "aws-access-token"
+description = "AWS"
+regex = '''(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}'''
+
+# Cryptographic keys
+[[rules]]
+id = "PKCS8-PK"
+description = "PKCS8 private key"
+regex = '''-----BEGIN PRIVATE KEY-----'''
+
+[[rules]]
+id = "RSA-PK"
+description = "RSA private key"
+regex = '''-----BEGIN RSA PRIVATE KEY-----'''
+
+[[rules]]
+id = "OPENSSH-PK"
+description = "SSH private key"
+regex = '''-----BEGIN OPENSSH PRIVATE KEY-----'''
+
+[[rules]]
+id = "PGP-PK"
+description = "PGP private key"
+regex = '''-----BEGIN PGP PRIVATE KEY BLOCK-----'''
+
+[[rules]]
+id = "github-pat"
+description = "GitHub Personal Access Token"
+regex = '''ghp_[0-9a-zA-Z]{36}'''
+
+[[rules]]
+id = "github-oauth"
+description = "GitHub OAuth Access Token"
+regex = '''gho_[0-9a-zA-Z]{36}'''
+
+[[rules]]
+id = "SSH-DSA-PK"
+description = "SSH (DSA) private key"
+regex = '''-----BEGIN DSA PRIVATE KEY-----'''
+
+[[rules]]
+id = "SSH-EC-PK"
+description = "SSH (EC) private key"
+regex = '''-----BEGIN EC PRIVATE KEY-----'''
+
+
+[[rules]]
+id = "github-app-token"
+description = "GitHub App Token"
+regex = '''(ghu|ghs)_[0-9a-zA-Z]{36}'''
+
+[[rules]]
+id = "github-refresh-token"
+description = "GitHub Refresh Token"
+regex = '''ghr_[0-9a-zA-Z]{76}'''
+
+[[rules]]
+id = "shopify-shared-secret"
+description = "Shopify shared secret"
+regex = '''shpss_[a-fA-F0-9]{32}'''
+
+[[rules]]
+id = "shopify-access-token"
+description = "Shopify access token"
+regex = '''shpat_[a-fA-F0-9]{32}'''
+
+[[rules]]
+id = "shopify-custom-access-token"
+description = "Shopify custom app access token"
+regex = '''shpca_[a-fA-F0-9]{32}'''
+
+[[rules]]
+id = "shopify-private-app-access-token"
+description = "Shopify private app access token"
+regex = '''shppa_[a-fA-F0-9]{32}'''
+
+[[rules]]
+id = "slack-access-token"
+description = "Slack token"
+regex = '''xox[baprs]-([0-9a-zA-Z]{10,48})?'''
+
+[[rules]]
+id = "stripe-access-token"
+description = "Stripe"
+regex = '''(?i)(sk|pk)_(test|live)_[0-9a-z]{10,32}'''
+
+[[rules]]
+id = "pypi-upload-token"
+description = "PyPI upload token"
+regex = '''pypi-AgEIcHlwaS5vcmc[A-Za-z0-9\-_]{50,1000}'''
+
+[[rules]]
+id = "gcp-service-account"
+description = "Google (GCP) Service-account"
+regex = '''\"type\": \"service_account\"'''
+
+[[rules]]
+id = "heroku-api-key"
+description = "Heroku API Key"
+regex = ''' (?i)(heroku[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([0-9A-F]{8}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{12})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "slack-web-hook"
+description = "Slack Webhook"
+regex = '''https://hooks.slack.com/services/T[a-zA-Z0-9_]{8}/B[a-zA-Z0-9_]{8,12}/[a-zA-Z0-9_]{24}'''
+
+[[rules]]
+id = "twilio-api-key"
+description = "Twilio API Key"
+regex = '''SK[0-9a-fA-F]{32}'''
+
+[[rules]]
+id = "age-secret-key"
+description = "Age secret key"
+regex = '''AGE-SECRET-KEY-1[QPZRY9X8GF2TVDW0S3JN54KHCE6MUA7L]{58}'''
+
+[[rules]]
+id = "facebook-token"
+description = "Facebook token"
+regex = '''(?i)(facebook[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-f0-9]{32})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "twitter-token"
+description = "Twitter token"
+regex = '''(?i)(twitter[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-f0-9]{35,44})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "adobe-client-id"
+description = "Adobe Client ID (Oauth Web)"
+regex = '''(?i)(adobe[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-f0-9]{32})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "adobe-client-secret"
+description = "Adobe Client Secret"
+regex = '''(p8e-)(?i)[a-z0-9]{32}'''
+
+[[rules]]
+id = "alibaba-access-key-id"
+description = "Alibaba AccessKey ID"
+regex = '''(LTAI)(?i)[a-z0-9]{20}'''
+
+[[rules]]
+id = "alibaba-secret-key"
+description = "Alibaba Secret Key"
+regex = '''(?i)(alibaba[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{30})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "asana-client-id"
+description = "Asana Client ID"
+regex = '''(?i)(asana[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([0-9]{16})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "asana-client-secret"
+description = "Asana Client Secret"
+regex = '''(?i)(asana[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{32})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "atlassian-api-token"
+description = "Atlassian API token"
+regex = '''(?i)(atlassian[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{24})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "bitbucket-client-id"
+description = "Bitbucket client ID"
+regex = '''(?i)(bitbucket[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{32})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "bitbucket-client-secret"
+description = "Bitbucket client secret"
+regex = '''(?i)(bitbucket[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9_\-]{64})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "beamer-api-token"
+description = "Beamer API token"
+regex = '''(?i)(beamer[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](b_[a-z0-9=_\-]{44})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "clojars-api-token"
+description = "Clojars API token"
+regex = '''(CLOJARS_)(?i)[a-z0-9]{60}'''
+
+[[rules]]
+id = "contentful-delivery-api-token"
+description = "Contentful delivery API token"
+regex = '''(?i)(contentful[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9\-=_]{43})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "databricks-api-token"
+description = "Databricks API token"
+regex = '''dapi[a-h0-9]{32}'''
+
+[[rules]]
+id = "discord-api-token"
+description = "Discord API key"
+regex = '''(?i)(discord[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-h0-9]{64})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "discord-client-id"
+description = "Discord client ID"
+regex = '''(?i)(discord[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([0-9]{18})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "discord-client-secret"
+description = "Discord client secret"
+regex = '''(?i)(discord[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9=_\-]{32})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "doppler-api-token"
+description = "Doppler API token"
+regex = '''['\"](dp\.pt\.)(?i)[a-z0-9]{43}['\"]'''
+
+[[rules]]
+id = "dropbox-api-secret"
+description = "Dropbox API secret/key"
+regex = '''(?i)(dropbox[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{15})['\"]'''
+
+[[rules]]
+id = "dropbox--api-key"
+description = "Dropbox API secret/key"
+regex = '''(?i)(dropbox[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{15})['\"]'''
+
+[[rules]]
+id = "dropbox-short-lived-api-token"
+description = "Dropbox short lived API token"
+regex = '''(?i)(dropbox[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](sl\.[a-z0-9\-=_]{135})['\"]'''
+
+[[rules]]
+id = "dropbox-long-lived-api-token"
+description = "Dropbox long lived API token"
+regex = '''(?i)(dropbox[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"][a-z0-9]{11}(AAAAAAAAAA)[a-z0-9\-_=]{43}['\"]'''
+
+[[rules]]
+id = "duffel-api-token"
+description = "Duffel API token"
+regex = '''['\"]duffel_(test|live)_(?i)[a-z0-9_-]{43}['\"]'''
+
+[[rules]]
+id = "dynatrace-api-token"
+description = "Dynatrace API token"
+regex = '''['\"]dt0c01\.(?i)[a-z0-9]{24}\.[a-z0-9]{64}['\"]'''
+
+[[rules]]
+id = "easypost-api-token"
+description = "EasyPost API token"
+regex = '''['\"]EZAK(?i)[a-z0-9]{54}['\"]'''
+
+[[rules]]
+id = "easypost-test-api-token"
+description = "EasyPost test API token"
+regex = '''['\"]EZTK(?i)[a-z0-9]{54}['\"]'''
+
+[[rules]]
+id = "fastly-api-token"
+description = "Fastly API token"
+regex = '''(?i)(fastly[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9\-=_]{32})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "finicity-client-secret"
+description = "Finicity client secret"
+regex = '''(?i)(finicity[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{20})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "finicity-api-token"
+description = "Finicity API token"
+regex = '''(?i)(finicity[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-f0-9]{32})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "flutterwave-public-key"
+description = "Flutterwave public key"
+regex = '''FLWPUBK_TEST-(?i)[a-h0-9]{32}-X'''
+
+[[rules]]
+id = "flutterwave-secret-key"
+description = "Flutterwave secret key"
+regex = '''FLWSECK_TEST-(?i)[a-h0-9]{32}-X'''
+
+[[rules]]
+id = "flutterwave-enc-key"
+description = "Flutterwave encrypted key"
+regex = '''FLWSECK_TEST[a-h0-9]{12}'''
+
+[[rules]]
+id = "frameio-api-token"
+description = "Frame.io API token"
+regex = '''fio-u-(?i)[a-z0-9\-_=]{64}'''
+
+[[rules]]
+id = "gocardless-api-token"
+description = "GoCardless API token"
+regex = '''['\"]live_(?i)[a-z0-9\-_=]{40}['\"]'''
+
+[[rules]]
+id = "grafana-api-token"
+description = "Grafana API token"
+regex = '''['\"]eyJrIjoi(?i)[a-z0-9\-_=]{72,92}['\"]'''
+
+[[rules]]
+id = "hashicorp-tf-api-token"
+description = "HashiCorp Terraform user/org API token"
+regex = '''['\"](?i)[a-z0-9]{14}\.atlasv1\.[a-z0-9\-_=]{60,70}['\"]'''
+
+[[rules]]
+id = "hubspot-api-token"
+description = "HubSpot API token"
+regex = '''(?i)(hubspot[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-h0-9]{8}-[a-h0-9]{4}-[a-h0-9]{4}-[a-h0-9]{4}-[a-h0-9]{12})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "intercom-api-token"
+description = "Intercom API token"
+regex = '''(?i)(intercom[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9=_]{60})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "intercom-client-secret"
+description = "Intercom client secret/ID"
+regex = '''(?i)(intercom[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-h0-9]{8}-[a-h0-9]{4}-[a-h0-9]{4}-[a-h0-9]{4}-[a-h0-9]{12})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "ionic-api-token"
+description = "Ionic API token"
+regex = '''(?i)(ionic[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](ion_[a-z0-9]{42})['\"]'''
+
+[[rules]]
+id = "linear-api-token"
+description = "Linear API token"
+regex = '''lin_api_(?i)[a-z0-9]{40}'''
+
+[[rules]]
+id = "linear-client-secret"
+description = "Linear client secret/ID"
+regex = '''(?i)(linear[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-f0-9]{32})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "lob-api-key"
+description = "Lob API Key"
+regex = '''(?i)(lob[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]((live|test)_[a-f0-9]{35})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "lob-pub-api-key"
+description = "Lob Publishable API Key"
+regex = '''(?i)(lob[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]((test|live)_pub_[a-f0-9]{31})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "mailchimp-api-key"
+description = "Mailchimp API key"
+regex = '''(?i)(mailchimp[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-f0-9]{32}-us20)['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "mailgun-private-api-token"
+description = "Mailgun private API token"
+regex = '''(?i)(mailgun[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](key-[a-f0-9]{32})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "mailgun-pub-key"
+description = "Mailgun public validation key"
+regex = '''(?i)(mailgun[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](pubkey-[a-f0-9]{32})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "mailgun-signing-key"
+description = "Mailgun webhook signing key"
+regex = '''(?i)(mailgun[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-h0-9]{32}-[a-h0-9]{8}-[a-h0-9]{8})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "mapbox-api-token"
+description = "Mapbox API token"
+regex = '''(?i)(pk\.[a-z0-9]{60}\.[a-z0-9]{22})'''
+
+[[rules]]
+id = "messagebird-api-token"
+description = "MessageBird API token"
+regex = '''(?i)(messagebird[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{25})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "messagebird-client-id"
+description = "MessageBird API client ID"
+regex = '''(?i)(messagebird[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-h0-9]{8}-[a-h0-9]{4}-[a-h0-9]{4}-[a-h0-9]{4}-[a-h0-9]{12})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "new-relic-user-api-key"
+description = "New Relic user API Key"
+regex = '''['\"](NRAK-[A-Z0-9]{27})['\"]'''
+
+[[rules]]
+id = "new-relic-user-api-id"
+description = "New Relic user API ID"
+regex = '''(?i)(newrelic[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([A-Z0-9]{64})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "new-relic-browser-api-token"
+description = "New Relic ingest browser API token"
+regex = '''['\"](NRJS-[a-f0-9]{19})['\"]'''
+
+[[rules]]
+id = "npm-access-token"
+description = "npm access token"
+regex = '''['\"](npm_(?i)[a-z0-9]{36})['\"]'''
+
+[[rules]]
+id = "planetscale-password"
+description = "PlanetScale password"
+regex = '''pscale_pw_(?i)[a-z0-9\-_\.]{43}'''
+
+[[rules]]
+id = "planetscale-api-token"
+description = "PlanetScale API token"
+regex = '''pscale_tkn_(?i)[a-z0-9\-_\.]{43}'''
+
+[[rules]]
+id = "postman-api-token"
+description = "Postman API token"
+regex = '''PMAK-(?i)[a-f0-9]{24}\-[a-f0-9]{34}'''
+
+[[rules]]
+id = "pulumi-api-token"
+description = "Pulumi API token"
+regex = '''pul-[a-f0-9]{40}'''
+
+[[rules]]
+id = "rubygems-api-token"
+description = "Rubygem API token"
+regex = '''rubygems_[a-f0-9]{48}'''
+
+[[rules]]
+id = "sendgrid-api-token"
+description = "SendGrid API token"
+regex = '''SG\.(?i)[a-z0-9_\-\.]{66}'''
+
+[[rules]]
+id = "sendinblue-api-token"
+description = "Sendinblue API token"
+regex = '''xkeysib-[a-f0-9]{64}\-(?i)[a-z0-9]{16}'''
+
+[[rules]]
+id = "shippo-api-token"
+description = "Shippo API token"
+regex = '''shippo_(live|test)_[a-f0-9]{40}'''
+
+[[rules]]
+id = "linkedin-client-secret"
+description = "LinkedIn Client secret"
+regex = '''(?i)(linkedin[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z]{16})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "linkedin-client-id"
+description = "LinkedIn Client ID"
+regex = '''(?i)(linkedin[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{14})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "twitch-api-token"
+description = "Twitch API token"
+regex = '''(?i)(twitch[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{30})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "typeform-api-token"
+description = "Typeform API token"
+regex = '''(?i)(typeform[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}(tfp_[a-z0-9\-_\.=]{59})'''
+secretGroup = 3
+
+[[rules]]
+id = "generic-api-key"
+description = "Generic API Key"
+regex = '''(?i)((key|api[^Version]|token|secret|password)[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([0-9a-zA-Z\-_=]{8,64})['\"]'''
+entropy = 3.7
+secretGroup = 4
+
+
+[allowlist]
+description = "global allow lists"
+regexes = ['''219-09-9999''', '''078-05-1120''', '''(9[0-9]{2}|666)-\d{2}-\d{4}''']
+paths = [
+    '''gitleaks.toml''',
+    '''(.*?)(jpg|gif|doc|pdf|bin|svg|socket)$''',
+    '''(go.mod|go.sum)$''',
+    
+    '''salt/nginx/files/enterprise-attack.json'''
+]

.github/workflows/leaktest.yml

@@ -13,3 +13,5 @@ jobs:
 
     - name: Gitleaks
       uses: zricethezav/gitleaks-action@master
+      with:
+        config-path: .github/.gitleaks.toml

.github/workflows/pythontest.yml

@@ -0,0 +1,31 @@
+name: python-test
+
+on: [push, pull_request]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10"]
+        python-code-path: ["salt/sensoroni/files/analyzers"]
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install flake8 pytest pytest-cov
+        find . -name requirements.txt -exec pip install -r {} \;
+    - name: Lint with flake8
+      run: |
+        flake8 ${{ matrix.python-code-path }} --show-source --max-complexity=12  --doctests --max-line-length=200 --statistics
+    - name: Test with pytest
+      run: |
+        pytest ${{ matrix.python-code-path }} --cov=${{ matrix.python-code-path }} --doctest-modules --cov-report=term --cov-fail-under=100 --cov-config=${{ matrix.python-code-path }}/pytest.ini

.gitignore

@@ -56,4 +56,15 @@ $RECYCLE.BIN/
 # Windows shortcuts
 *.lnk
 
-# End of https://www.gitignore.io/api/macos,windows
+# End of https://www.gitignore.io/api/macos,windows
+
+# Pytest output
+__pycache__
+.pytest_cache
+.coverage
+*.pyc
+.venv
+
+# Analyzer dev/test config files
+*_dev.yaml
+site-packages

CONTRIBUTING.md

@@ -29,7 +29,11 @@
 
 * See this document's [code styling and conventions section](#code-style-and-conventions) below to be sure your PR fits our code requirements prior to submitting.
 
-* Minor bug fixes can be submitted immediately. However, if you are wanting to make more involved changes, please start a [discussion](https://github.com/Security-Onion-Solutions/securityonion/discussions) first and tell us what you are hoping to achieve. If we agree with your goals, then you can submit the PR.
+* Change behavior (fix a bug, add a new feature) separately from refactoring code. Refactor pull requests are welcome, but ensure your new code behaves exactly the same as the old.
+
+* **Do not refactor code for non-functional reasons**. If you are submitting a pull request that refactors code, ensure the refactor is improving the functionality of the code you're refactoring (e.g. decreasing complexity, removing reliance on 3rd party tools, improving performance).
+
+* Before submitting a PR with significant changes to the project, [start a discussion](https://github.com/Security-Onion-Solutions/securityonion/discussions/new) explaining what you hope to acheive. The project maintainers will provide feedback and determine whether your goal aligns with the project. 
 
 
 ### Code style and conventions
@@ -38,3 +42,5 @@
 * All new Bash code should pass [ShellCheck](https://www.shellcheck.net/) analysis. Where errors can be *safely* [ignored](https://github.com/koalaman/shellcheck/wiki/Ignore), the relevant disable directive should be accompanied by a brief explanation as to why the error is being ignored.
 
 * **Ensure all YAML (this includes Salt states and pillars) is properly formatted**. The spec for YAML v1.2 can be found [here](https://yaml.org/spec/1.2/spec.html), however there are numerous online resources with simpler descriptions of its formatting rules. 
+
+* **All code of any language should match the style of other code of that same language within the project.** Be sure that any changes you make do not break from the pre-existing style of Security Onion code.

HOTFIX

@@ -1 +1 @@
-
+

README.md

@@ -1,14 +1,20 @@
-## Security Onion 2.3.100
+## Security Onion 2.3.130
 
-Security Onion 2.3.100 is here!
+Security Onion 2.3.130 is here!
 
 ## Screenshots
 
 Alerts
-![Alerts](./assets/images/screenshots/alerts-1.png)
+![Alerts](./assets/images/screenshots/alerts.png)
+
+Dashboards
+![Dashboards](./assets/images/screenshots/dashboards.png)
 
 Hunt
-![Hunt](./assets/images/screenshots/hunt-1.png)
+![Hunt](./assets/images/screenshots/hunt.png)
+
+Cases
+![Cases](./assets/images/screenshots/cases-comments.png)
 
 ### Release Notes
 

VERIFY_ISO.md

@@ -1,18 +1,18 @@
-### 2.3.100-20220131 ISO image built on 2022/01/31
+### 2.3.130-20220607 ISO image built on 2022/06/07
 
 
 
 ### Download and Verify
 
-2.3.100-20220131 ISO image:  
-https://download.securityonion.net/file/securityonion/securityonion-2.3.100-20220131.iso
+2.3.130-20220607 ISO image:  
+https://download.securityonion.net/file/securityonion/securityonion-2.3.130-20220607.iso
 
-MD5: 9B50774532B77A10E2F52A3F0492A780  
-SHA1: 3C50D2EF4AFFFA8929492C2FC3842FF3EEE0EA5F  
-SHA256: CDCBEE6B1FDFB4CAF6C9F80CCADC161366EC337746E8394BF4454FAA2FC11AA1 
+MD5: 0034D6A9461C04357AFF512875408A4C  
+SHA1: BF80EEB101C583153CAD8E185A7DB3173FD5FFE8  
+SHA256: 15943623B96D8BB4A204A78668447F36B54A63ABA5F8467FBDF0B25C5E4E6078 
 
 Signature for ISO image:  
-https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.100-20220131.iso.sig
+https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.130-20220607.iso.sig
 
 Signing key:  
 https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS  
@@ -26,22 +26,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/ma
 
 Download the signature file for the ISO:  
 ```
-wget https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.100-20220131.iso.sig
+wget https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.130-20220607.iso.sig
 ```
 
 Download the ISO image:  
 ```
-wget https://download.securityonion.net/file/securityonion/securityonion-2.3.100-20220131.iso
+wget https://download.securityonion.net/file/securityonion/securityonion-2.3.130-20220607.iso
 ```
 
 Verify the downloaded ISO image using the signature file:  
 ```
-gpg --verify securityonion-2.3.100-20220131.iso.sig securityonion-2.3.100-20220131.iso
+gpg --verify securityonion-2.3.130-20220607.iso.sig securityonion-2.3.130-20220607.iso
 ```
 
 The output should show "Good signature" and the Primary key fingerprint should match what's shown below:
 ```
-gpg: Signature made Mon 31 Jan 2022 11:41:30 AM EST using RSA key ID FE507013
+gpg: Signature made Tue 07 Jun 2022 01:27:20 PM EDT using RSA key ID FE507013
 gpg: Good signature from "Security Onion Solutions, LLC <info@securityonionsolutions.com>"
 gpg: WARNING: This key is not certified with a trusted signature!
 gpg:          There is no indication that the signature belongs to the owner.

VERSION

@@ -1 +1 @@
-2.3.100
+2.3.130

files/firewall/assigned_hostgroups.local.map.yaml

@@ -13,6 +13,7 @@ role:
   fleet:
   heavynode:
   helixsensor:
+  idh:
   import:
   manager:
   managersearch:

files/firewall/hostgroups.local.yaml

@@ -28,6 +28,10 @@ firewall:
       ips:
         delete:
         insert:
+    idh:
+      ips:
+        delete:
+        insert: 
     manager:
       ips:
         delete:

pillar/elasticsearch/eval.sls

@@ -1,14 +1,2 @@
 elasticsearch:
   templates:
-    - so/so-beats-template.json.jinja
-    - so/so-case-template.json.jinja
-    - so/so-common-template.json.jinja
-    - so/so-firewall-template.json.jinja
-    - so/so-flow-template.json.jinja
-    - so/so-ids-template.json.jinja
-    - so/so-import-template.json.jinja
-    - so/so-osquery-template.json.jinja
-    - so/so-ossec-template.json.jinja
-    - so/so-strelka-template.json.jinja
-    - so/so-syslog-template.json.jinja
-    - so/so-zeek-template.json.jinja

pillar/elasticsearch/index_templates.sls

@@ -0,0 +1,2 @@
+elasticsearch:
+  index_settings:

pillar/elasticsearch/manager.sls

@@ -1,15 +1,2 @@
 elasticsearch:
   templates:
-    - so/so-beats-template.json.jinja
-    - so/so-case-template.json.jinja
-    - so/so-common-template.json.jinja
-    - so/so-endgame-template.json.jinja
-    - so/so-firewall-template.json.jinja
-    - so/so-flow-template.json.jinja
-    - so/so-ids-template.json.jinja
-    - so/so-import-template.json.jinja
-    - so/so-osquery-template.json.jinja
-    - so/so-ossec-template.json.jinja
-    - so/so-strelka-template.json.jinja
-    - so/so-syslog-template.json.jinja
-    - so/so-zeek-template.json.jinja

pillar/elasticsearch/search.sls

@@ -1,15 +1,2 @@
 elasticsearch:
   templates:
-    - so/so-beats-template.json.jinja
-    - so/so-case-template.json.jinja
-    - so/so-common-template.json.jinja
-    - so/so-endgame-template.json.jinja
-    - so/so-firewall-template.json.jinja
-    - so/so-flow-template.json.jinja
-    - so/so-ids-template.json.jinja
-    - so/so-import-template.json.jinja
-    - so/so-osquery-template.json.jinja
-    - so/so-ossec-template.json.jinja
-    - so/so-strelka-template.json.jinja
-    - so/so-syslog-template.json.jinja
-    - so/so-zeek-template.json.jinja

pillar/logstash/nodes.sls

@@ -1,11 +1,13 @@
 {% set node_types = {} %}
+{% set cached_grains = salt.saltutil.runner('cache.grains', tgt='*') %}
 {% for minionid, ip in salt.saltutil.runner(
     'mine.get',
-    tgt='G@role:so-manager or G@role:so-managersearch or G@role:so-standalone or G@role:so-node or G@role:so-heavynode or G@role:so-receiver or G@role:so-helix ',
+    tgt='G@role:so-manager or G@role:so-managersearch or G@role:so-standalone or G@role:so-node or G@role:so-heavynode or G@role:so-receiver or G@role:so-helix',
     fun='network.ip_addrs',
-    tgt_type='compound') | dictsort() 
+    tgt_type='compound') | dictsort()
 %}
-{%   set hostname = minionid.split('_')[0] %}
+
+{%   set hostname = cached_grains[minionid]['host'] %}
 {%   set node_type = minionid.split('_')[1] %}
 {%   if node_type not in node_types.keys() %}
 {%     do node_types.update({node_type: {hostname: ip[0]}}) %}

pillar/logstash/search.sls

@@ -13,4 +13,5 @@ logstash:
         - so/9600_output_ossec.conf.jinja
         - so/9700_output_strelka.conf.jinja
         - so/9800_output_logscan.conf.jinja
+        - so/9801_output_rita.conf.jinja
         - so/9900_output_endgame.conf.jinja

pillar/top.sls

@@ -15,12 +15,12 @@ base:
     - logstash
     - logstash.manager
     - logstash.search
-    - elasticsearch.search
+    - elasticsearch.index_templates
 
   '*_manager':
     - logstash
     - logstash.manager
-    - elasticsearch.manager
+    - elasticsearch.index_templates
 
   '*_manager or *_managersearch':
     - match: compound
@@ -46,7 +46,7 @@ base:
     - zeeklogs
     - secrets
     - healthcheck.eval
-    - elasticsearch.eval
+    - elasticsearch.index_templates
 {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
     - elasticsearch.auth
 {% endif %}
@@ -60,7 +60,7 @@ base:
     - logstash
     - logstash.manager
     - logstash.search
-    - elasticsearch.search
+    - elasticsearch.index_templates
 {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
     - elasticsearch.auth
 {% endif %}
@@ -98,10 +98,15 @@ base:
     - global
     - minions.{{ grains.id }}
 
+  '*_idh':
+    - data.*
+    - global
+    - minions.{{ grains.id }}
+
   '*_searchnode':
     - logstash
     - logstash.search
-    - elasticsearch.search
+    - elasticsearch.index_templates
     - elasticsearch.auth
     - global
     - minions.{{ grains.id }}
@@ -117,7 +122,7 @@ base:
   '*_import':
     - zeeklogs
     - secrets
-    - elasticsearch.eval
+    - elasticsearch.index_templates
 {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
     - elasticsearch.auth
 {% endif %}
@@ -126,3 +131,6 @@ base:
 {% endif %}
     - global
     - minions.{{ grains.id }}
+
+  '*_workstation':
+    - minions.{{ grains.id }}

salt/allowed_states.map.jinja

@@ -1,6 +1,5 @@
 {% set ZEEKVER = salt['pillar.get']('global:mdengine', '') %}
 {% set WAZUH = salt['pillar.get']('global:wazuh', '0') %}
-{% set THEHIVE = salt['pillar.get']('manager:thehive', '0') %}
 {% set PLAYBOOK = salt['pillar.get']('manager:playbook', '0') %}
 {% set FREQSERVER = salt['pillar.get']('manager:freq', '0') %}
 {% set DOMAINSTATS = salt['pillar.get']('manager:domainstats', '0') %}
@@ -91,6 +90,16 @@
           'schedule',
           'docker_clean'
           ],
+     'so-idh': [
+          'ssl',
+          'telegraf',
+          'firewall',
+          'fleet.install_package',
+          'filebeat',
+          'idh',
+          'schedule',
+          'docker_clean'
+          ],
       'so-import': [
           'salt.master',
           'ca',
@@ -208,6 +217,8 @@
           'schedule',
           'docker_clean'
           ],
+      'so-workstation': [
+          ],
   }, grain='role') %}
 
   {% if FILEBEAT and grains.role in ['so-helixsensor', 'so-eval', 'so-manager', 'so-standalone', 'so-node', 'so-managersearch', 'so-heavynode', 'so-import', 'so-receiver'] %}
@@ -238,7 +249,7 @@
     {% do allowed_states.append('strelka') %}
   {% endif %}
 
-  {% if WAZUH and grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-node', 'so-managersearch', 'so-heavynode', 'so-receiver']%}
+  {% if WAZUH and grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-node', 'so-managersearch', 'so-heavynode', 'so-receiver','so-idh']%}
     {% do allowed_states.append('wazuh') %}
   {% endif %}
 
@@ -263,10 +274,6 @@
     {% do allowed_states.append('elastalert') %}
   {% endif %}
 
-  {% if (THEHIVE != 0) and grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-managersearch'] %}
-    {% do allowed_states.append('thehive') %}
-  {% endif %}
-
   {% if (PLAYBOOK !=0) and grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-managersearch'] %}
     {% do allowed_states.append('playbook') %}
   {% endif %}

salt/common/files/log-rotate.conf

@@ -23,6 +23,7 @@
 /opt/so/log/salt/minion
 /opt/so/log/salt/master
 /opt/so/log/logscan/*.log
+/nsm/idh/*.log
 {
     {{ logrotate_conf | indent(width=4) }}
 }

salt/common/init.sls

@@ -300,8 +300,17 @@ sostatus_log:
     - month: '*'
     - dayweek: '*'
 
-
 {% if role in ['eval', 'manager', 'managersearch', 'standalone'] %}
+# Install cron job to determine size of influxdb for telegraf
+'du -s -k /nsm/influxdb | cut -f1 > /opt/so/log/telegraf/influxdb_size.log 2>&1':
+  cron.present:
+    - user: root
+    - minute: '*/1'
+    - hour: '*'
+    - daymonth: '*'
+    - month: '*'
+    - dayweek: '*'
+    
 # Lock permissions on the backup directory
 backupdir:
   file.directory:

salt/common/tools/sbin/so-analyst-install

@@ -15,295 +15,86 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-if [ "$(id -u)" -ne 0 ]; then
-	echo "This script must be run using sudo!"
-	exit 1
-fi
+doc_workstation_url="https://docs.securityonion.net/en/2.3/analyst-vm.html"
+{# we only want the script to install the workstation if it is CentOS -#}
+{% if grains.os == 'CentOS' -%}
+{#   if this is a manager -#}
+{%   if grains.master == grains.id.split('_')|first -%}
 
-INSTALL_LOG=/root/so-analyst-install.log
-exec &> >(tee -a "$INSTALL_LOG")
+source /usr/sbin/so-common
+pillar_file="/opt/so/saltstack/local/pillar/minions/{{grains.id}}.sls"
 
-log() {
-	msg=$1
-	level=${2:-I}
-	now=$(TZ=GMT date +"%Y-%m-%dT%H:%M:%SZ")
-	echo -e "$now | $level | $msg" >> "$INSTALL_LOG" 2>&1
-}
+if [ -f "$pillar_file" ]; then
+  if ! grep -q "^workstation:$" "$pillar_file"; then
 
-error() {
-	log "$1" "E"
-}
-
-info() {
-	log "$1" "I"
-}
-
-title() {
-	echo -e "\n-----------------------------\n $1\n-----------------------------\n" >> "$INSTALL_LOG" 2>&1
-}
-
-logCmd() {
-	cmd=$1
-	info "Executing command: $cmd"
-	$cmd >> "$INSTALL_LOG" 2>&1
-}
-
-analyze_system() {
-	title "System Characteristics"
-	logCmd "uptime"
-	logCmd "uname -a"
-	logCmd "free -h"
-	logCmd "lscpu"
-	logCmd "df -h"
-	logCmd "ip a"
-}
-
-analyze_system
-
-OS=$(grep PRETTY_NAME /etc/os-release | grep 'CentOS Linux 7')
-if [ $? -ne 0 ]; then
-  echo "This is an unsupported OS. Please use CentOS 7 to install the analyst node."
-  exit 1
-fi
-
-if [[ "$manufacturer" == "Security Onion Solutions" && "$family" == "Automated" ]]; then
-  INSTALL=yes
-  CURLCONTINUE=no
-else
-  INSTALL=''
-  CURLCONTINUE=''
-fi
-
-FIRSTPASS=yes
-while [[ $INSTALL != "yes" ]] && [[ $INSTALL != "no" ]]; do
-  if [[ "$FIRSTPASS" == "yes" ]]; then
-    clear
-    echo "###########################################"
-    echo "##          ** W A R N I N G **          ##"
-    echo "##    _______________________________    ##"
-    echo "##                                       ##"
-    echo "##    Installing the Security Onion      ##"
-    echo "##   analyst node on this device will    ##"
-    echo "##       make permanent changes to       ##"
-    echo "##              the system.              ##"
-    echo "##                                       ##"
-    echo "###########################################"
-    echo "Do you wish to continue? (Type the entire word 'yes' to proceed or 'no' to exit)"
-    FIRSTPASS=no
-  else
-    echo "Please type 'yes' to continue or 'no' to exit."
-  fi      
-  read INSTALL
-done
-
-if [[ $INSTALL == "no" ]]; then
-  echo "Exiting analyst node installation."
-  exit 0
-fi
-
-echo "Testing for internet connection with curl https://securityonionsolutions.com/"
-CANCURL=$(curl -sI https://securityonionsolutions.com/ | grep "200 OK")
-  if [ $? -ne 0 ]; then
     FIRSTPASS=yes
-    while [[ $CURLCONTINUE != "yes" ]] && [[ $CURLCONTINUE != "no" ]]; do
+    while [[ $INSTALL != "yes" ]] && [[ $INSTALL != "no" ]]; do
       if [[ "$FIRSTPASS" == "yes" ]]; then
-        echo "We could not access https://securityonionsolutions.com/."
-        echo "Since packages are downloaded from the internet, internet access is required."
-        echo "If you would like to ignore this warning and continue anyway, please type 'yes'."
-        echo "Otherwise, type 'no' to exit."
+        echo "###########################################"
+        echo "##          ** W A R N I N G **          ##"
+        echo "##    _______________________________    ##"
+        echo "##                                       ##"
+        echo "##    Installing the Security Onion      ##"
+        echo "##   analyst node on this device will    ##"
+        echo "##       make permanent changes to       ##"
+        echo "##              the system.              ##"
+        echo "##    A system reboot will be required   ##"
+        echo "##        to complete the install.       ##"
+        echo "##                                       ##"
+        echo "###########################################"
+        echo "Do you wish to continue? (Type the entire word 'yes' to proceed or 'no' to exit)"
         FIRSTPASS=no
       else
         echo "Please type 'yes' to continue or 'no' to exit."
-      fi  
-      read CURLCONTINUE
+      fi      
+      read INSTALL
     done
-    if [[ "$CURLCONTINUE" == "no" ]]; then
+
+    if [[ $INSTALL == "no" ]]; then
       echo "Exiting analyst node installation."
       exit 0
     fi
-  else
-    echo "We were able to curl https://securityonionsolutions.com/."
-    sleep 3
+
+    # Add workstation pillar to the minion's pillar file
+    printf '%s\n'\
+      "workstation:"\
+      "  gui:"\
+      "    enabled: true"\
+		  "" >> "$pillar_file"
+    echo "Applying the workstation state. This could take some time since there are many packages that need to be installed."
+    if salt-call state.apply workstation -linfo queue=True; then # make sure the state ran successfully
+      echo ""
+      echo "Analyst workstation has been installed!"
+      echo "Press ENTER to reboot or Ctrl-C to cancel."
+      read pause
+
+      reboot;
+    else
+      echo "There was an issue applying the workstation state. Please review the log above or at /opt/so/logs/salt/minion."
+    fi
+  else # workstation is already added
+    echo "The workstation pillar already exists in $pillar_file."
+    echo "To enable/disable the gui, set 'workstation:gui:enabled' to true or false in $pillar_file."
+    echo "Additional documentation can be found at $doc_workstation_url."
   fi
-
-# Install a GUI text editor
-yum -y install gedit
-
-# Install misc utils
-yum -y install wget curl unzip epel-release yum-plugin-versionlock;
-
-# Install xWindows
-yum -y groupinstall "X Window System";
-yum -y install gnome-classic-session gnome-terminal nautilus-open-terminal control-center liberation-mono-fonts;
-unlink /etc/systemd/system/default.target;
-ln -sf /lib/systemd/system/graphical.target /etc/systemd/system/default.target;
-yum -y install file-roller
-
-# Install Mono - prereq for NetworkMiner
-yum -y install mono-core mono-basic mono-winforms expect
-
-# Install NetworkMiner
-yum -y install libcanberra-gtk2;
-wget https://www.netresec.com/?download=NetworkMiner -O /tmp/nm.zip;
-mkdir -p /opt/networkminer/
-unzip /tmp/nm.zip -d /opt/networkminer/;
-rm /tmp/nm.zip;
-mv /opt/networkminer/NetworkMiner_*/* /opt/networkminer/
-chmod +x /opt/networkminer/NetworkMiner.exe;
-chmod -R go+w /opt/networkminer/AssembledFiles/;
-chmod -R go+w /opt/networkminer/Captures/;
-# Create networkminer shim
-cat << EOF >> /bin/networkminer
-#!/bin/bash
-/bin/mono /opt/networkminer/NetworkMiner.exe --noupdatecheck "\$@"
-EOF
-chmod +x /bin/networkminer
-# Convert networkminer ico file to png format
-yum -y install ImageMagick
-convert /opt/networkminer/networkminericon.ico /opt/networkminer/networkminericon.png
-# Create menu entry
-cat << EOF >> /usr/share/applications/networkminer.desktop
-[Desktop Entry]
-Name=NetworkMiner
-Comment=NetworkMiner
-Encoding=UTF-8
-Exec=/bin/networkminer %f
-Icon=/opt/networkminer/networkminericon-4.png
-StartupNotify=true
-Terminal=false
-X-MultipleArgs=false
-Type=Application
-MimeType=application/x-pcap;
-Categories=Network;
-EOF
-
-# Set default monospace font to Liberation
-cat << EOF >> /etc/fonts/local.conf
- <match target="pattern">
-  <test name="family" qual="any">
-   <string>monospace</string>
-  </test>
-  <edit binding="strong" mode="prepend" name="family">
-   <string>Liberation Mono</string>
-  </edit>
- </match>
-EOF
-
-# Install Wireshark for Gnome
-yum -y install wireshark-gnome; 
-
-# Install dnsiff
-yum -y install dsniff;
-
-# Install hping3
-yum -y install hping3;
-
-# Install netsed
-yum -y install netsed;
-
-# Install ngrep
-yum -y install ngrep;
-
-# Install scapy
-yum -y install python36-scapy;
-
-# Install ssldump
-yum -y install ssldump;
-
-# Install tcpdump
-yum -y install tcpdump;
-
-# Install tcpflow
-yum -y install tcpflow;
-
-# Install tcpxtract
-yum -y install tcpxtract;
-
-# Install whois
-yum -y install whois;
-
-# Install foremost
-yum -y install https://forensics.cert.org/centos/cert/7/x86_64//foremost-1.5.7-13.1.el7.x86_64.rpm;
-
-# Install chromium
-yum -y install chromium;
-
-# Install tcpstat
-yum -y install https://github.com/Security-Onion-Solutions/securityonion-docker-rpm/releases/download/securityonion-tcpstat-1.5.0/securityonion-tcpstat-1.5.0.rpm;
-
-# Install tcptrace
-yum -y install https://github.com/Security-Onion-Solutions/securityonion-docker-rpm/releases/download/securityonion-tcptrace-6.6.7/securityonion-tcptrace-6.6.7.rpm;
-
-# Install sslsplit
-yum -y install libevent;
-yum -y install sslsplit;
-
-# Install Bit-Twist
-yum -y install https://github.com/Security-Onion-Solutions/securityonion-docker-rpm/releases/download/securityonion-bittwist-2.0.0/securityonion-bittwist-2.0.0.rpm;
-
-# Install chaosreader
-yum -y install perl-IO-Compress perl-Net-DNS;
-yum -y install https://github.com/Security-Onion-Solutions/securityonion-docker-rpm/releases/download/securityonion-chaosreader-0.95.10/securityonion-chaosreader-0.95.10.rpm;
-chmod +x /bin/chaosreader;
-
-if [ -f ../../files/analyst/README ]; then
-  cp ../../files/analyst/README /;
-  cp ../../files/analyst/so-wallpaper.jpg /usr/share/backgrounds/;
-  cp ../../files/analyst/so-lockscreen.jpg /usr/share/backgrounds/;
-  cp ../../files/analyst/so-login-logo-dark.svg /usr/share/pixmaps/;
-else
-  cp /opt/so/saltstack/default/salt/common/files/analyst/README /;
-  cp /opt/so/saltstack/default/salt/common/files/analyst/so-wallpaper.jpg /usr/share/backgrounds/;
-  cp /opt/so/saltstack/default/salt/common/files/analyst/so-lockscreen.jpg /usr/share/backgrounds/;
-  cp /opt/so/saltstack/default/salt/common/files/analyst/so-login-logo-dark.svg /usr/share/pixmaps/;
+else # if the pillar file doesn't exist
+  echo "Could not find $pillar_file and add the workstation pillar."
 fi
 
-# Set background wallpaper
-cat << EOF >> /etc/dconf/db/local.d/00-background
-# Specify the dconf path
-[org/gnome/desktop/background]
+{#-  if this is not a manager #}
+{%   else -%}
 
-# Specify the path to the desktop background image file
-picture-uri='file:///usr/share/backgrounds/so-wallpaper.jpg'
-# Specify one of the rendering options for the background image:
-# 'none', 'wallpaper', 'centered', 'scaled', 'stretched', 'zoom', 'spanned'
-picture-options='zoom'
-# Specify the left or top color when drawing gradients or the solid color
-primary-color='000000'
-# Specify the right or bottom color when drawing gradients
-secondary-color='FFFFFF'
-EOF
+echo "Since this is not a manager, the pillar values to enable analyst workstation must be set manually. Please view the documentation at $doc_workstation_url."
 
-# Set lock screen
-cat << EOF >> /etc/dconf/db/local.d/00-screensaver
-[org/gnome/desktop/session]
-idle-delay=uint32 180
+{#- endif if this is a manager #}
+{%   endif -%}
 
-[org/gnome/desktop/screensaver]
-lock-enabled=true
-lock-delay=uint32 120
-picture-options='zoom'
-picture-uri='file:///usr/share/backgrounds/so-lockscreen.jpg'
-EOF
+{#- if not CentOS #}
+{%- else %}
 
-cat << EOF >> /etc/dconf/db/local.d/locks/screensaver
-/org/gnome/desktop/session/idle-delay
-/org/gnome/desktop/screensaver/lock-enabled
-/org/gnome/desktop/screensaver/lock-delay
-EOF
+echo "The Analyst Workstation can only be installed on CentOS. Please view the documentation at $doc_workstation_url."
 
-# Do not show the user list at login screen
-cat << EOF >> /etc/dconf/db/local.d/00-login-screen
-[org/gnome/login-screen]
-logo='/usr/share/pixmaps/so-login-logo-dark.svg'
-disable-user-list=true
-EOF
+{#- endif grains.os == CentOS #}
+{% endif -%}
 
-dconf update;
-
-echo
-echo "Analyst workstation has been installed!"
-echo "Press ENTER to reboot or Ctrl-C to cancel."
-read pause
-
-reboot;
+exit 0

salt/common/tools/sbin/so-common

@@ -120,6 +120,30 @@ check_elastic_license() {
 	fi  
 }
 
+check_salt_master_status() {
+	local timeout=$1
+	echo "Checking if we can talk to the salt master"
+	salt-call state.show_top concurrent=true
+
+	return
+}
+
+check_salt_minion_status() {
+	local timeout=$1
+	echo "Checking if the salt minion will respond to jobs" >> "$setup_log" 2>&1
+	salt "$MINION_ID" test.ping -t $timeout > /dev/null 2>&1
+	local status=$?
+	if [ $status -gt 0 ]; then
+		echo "  Minion did not respond" >> "$setup_log" 2>&1
+	else
+		echo "  Received job response from salt minion" >> "$setup_log" 2>&1
+	fi
+
+	return $status
+}
+
+
+
 copy_new_files() {
   # Copy new files over to the salt dir
   cd $UPDATE_DIR
@@ -249,6 +273,7 @@ lookup_salt_value() {
 	group=$2
 	kind=$3
 	output=${4:-newline_values_only}
+	local=$5
 
 	if [ -z "$kind" ]; then
 		kind=pillar
@@ -258,7 +283,13 @@ lookup_salt_value() {
 		group=${group}:
 	fi
 
-	salt-call --no-color  ${kind}.get ${group}${key} --out=${output}
+	if [[ "$local" == "--local" ]] || [[ "$local" == "local" ]]; then
+		local="--local"
+	else
+		local=""
+	fi
+
+	salt-call --no-color  ${kind}.get ${group}${key} --out=${output} ${local}
 }
 
 lookup_pillar() {
@@ -360,6 +391,7 @@ run_check_net_err() {
 		exit $exit_code
 	fi
 }
+
 set_cron_service_name() {
   if [[ "$OS" == "centos" ]]; then
     cron_service_name="crond"

salt/common/tools/sbin/so-cortex-restart

@@ -17,5 +17,4 @@
 
 . /usr/sbin/so-common
 
-/usr/sbin/so-stop cortex $1
-/usr/sbin/so-start thehive $1
+echo "TheHive and its components are no longer part of Security Onion"

salt/common/tools/sbin/so-cortex-start

@@ -17,4 +17,4 @@
 
 . /usr/sbin/so-common
 
-/usr/sbin/so-start thehive $1
+echo "TheHive and its components are no longer part of Security Onion"

salt/common/tools/sbin/so-cortex-stop

@@ -17,4 +17,4 @@
 
 . /usr/sbin/so-common
 
-/usr/sbin/so-stop cortex $1
+echo "TheHive and its components are no longer part of Security Onion"

salt/common/tools/sbin/so-cortex-user-add

@@ -17,38 +17,4 @@
 
 . /usr/sbin/so-common
 
-usage() {
-    echo "Usage: $0 <new-user-name>"
-    echo ""
-    echo "Adds a new user to Cortex. The new password will be read from STDIN."
-    exit 1
-}
-
-if [ $# -ne 1 ]; then
-  usage
-fi
-
-USER=$1
-
-CORTEX_KEY=$(lookup_pillar cortexorguserkey)
-CORTEX_API_URL="$(lookup_pillar url_base)/cortex/api"
-CORTEX_ORG_NAME=$(lookup_pillar cortexorgname)
-CORTEX_USER=$USER
-
-# Read password for new user from stdin
-test -t 0
-if [[ $? == 0 ]]; then
-  echo "Enter new password:"
-fi
-read -rs CORTEX_PASS
-
-# Create new user in Cortex
-resp=$(curl -sk -XPOST -H "Authorization: Bearer $CORTEX_KEY" -H "Content-Type: application/json" -L "https://$CORTEX_API_URL/user" -d "{\"name\": \"$CORTEX_USER\",\"roles\": [\"read\",\"analyze\",\"orgadmin\"],\"organization\": \"$CORTEX_ORG_NAME\",\"login\": \"$CORTEX_USER\",\"password\" : \"$CORTEX_PASS\" }")
-if [[ "$resp" =~ \"status\":\"Ok\" ]]; then
-    echo "Successfully added user to Cortex."
-else
-    echo "Unable to add user to Cortex; user might already exist."
-    echo $resp
-    exit 2
-fi
-    
+echo "TheHive and its components are no longer part of Security Onion"

salt/common/tools/sbin/so-cortex-user-enable

@@ -17,41 +17,4 @@
 
 . /usr/sbin/so-common
 
-usage() {
-    echo "Usage: $0 <user-name> <true|false>"
-    echo ""
-    echo "Enables or disables a user in Cortex."
-    exit 1
-}
-
-if [ $# -ne 2 ]; then
-  usage
-fi
-
-USER=$1
-
-CORTEX_KEY=$(lookup_pillar cortexorguserkey)
-CORTEX_API_URL="$(lookup_pillar url_base)/cortex/api"
-CORTEX_USER=$USER
-
-case "${2^^}" in
-	FALSE | NO | 0)
-		CORTEX_STATUS=Locked
-		;;
-	TRUE | YES | 1)
-		CORTEX_STATUS=Ok
-		;;
-	*)
-		usage
-		;;
-esac
-
-resp=$(curl -sk -XPATCH -H "Authorization: Bearer $CORTEX_KEY" -H "Content-Type: application/json" -L "https://$CORTEX_API_URL/user/${CORTEX_USER}" -d "{\"status\":\"${CORTEX_STATUS}\" }")
-if [[ "$resp" =~ \"status\":\"Locked\" || "$resp" =~ \"status\":\"Ok\" ]]; then
-    echo "Successfully updated user in Cortex."
-else
-    echo "Failed to update user in Cortex."
-    echo $resp
-    exit 2
-fi
-    
+echo "TheHive and its components are no longer part of Security Onion"

salt/common/tools/sbin/so-elasticsearch-component-templates-list

@@ -0,0 +1,23 @@
+#!/bin/bash
+#
+# Copyright 2014-2022 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+{%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%}
+. /usr/sbin/so-common
+if [ "$1" == "" ]; then
+    {{ ELASTICCURL }} -s -k -L https://{{ NODEIP }}:9200/_component_template | jq '.component_templates[] |.name'| sort
+else
+    {{ ELASTICCURL }} -s -k -L https://{{ NODEIP }}:9200/_component_template/$1 | jq
+fi

salt/common/tools/sbin/so-elasticsearch-index-templates-list

@@ -0,0 +1,23 @@
+#!/bin/bash
+#
+# Copyright 2014-2022 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+{%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%}
+. /usr/sbin/so-common
+if [ "$1" == "" ]; then
+    {{ ELASTICCURL }} -s -k -L https://{{ NODEIP }}:9200/_index_template | jq '.index_templates[] |.name'| sort
+else
+    {{ ELASTICCURL }} -s -k -L https://{{ NODEIP }}:9200/_index_template/$1 | jq
+fi

salt/common/tools/sbin/so-elasticsearch-indices-list

@@ -18,4 +18,4 @@
 
 . /usr/sbin/so-common
 
-{{ ELASTICCURL }} -s -k -L https://{{ NODEIP }}:9200/_cat/indices?pretty
+{{ ELASTICCURL }} -s -k -L "https://{{ NODEIP }}:9200/_cat/indices?pretty&v&s=index"

salt/common/tools/sbin/so-elasticsearch-indices-rw

@@ -17,9 +17,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 IP={{ salt['grains.get']('ip_interfaces').get(salt['pillar.get']('sensor:mainint', salt['pillar.get']('manager:mainint', salt['pillar.get']('elasticsearch:mainint', salt['pillar.get']('host:mainint')))))[0] }}
 ESPORT=9200
-THEHIVEESPORT=9400
 
 echo "Removing read only attributes for indices..."
 echo
 {{ ELASTICCURL }} -s -k -XPUT -H "Content-Type: application/json" -L https://$IP:9200/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}' 2>&1 | if grep -q ack; then echo "Index settings updated..."; else echo "There was any issue updating the read-only attribute.  Please ensure Elasticsearch is running.";fi;
-{{ ELASTICCURL }} -XPUT -H "Content-Type: application/json" -L http://$IP:9400/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}' 2>&1 | if grep -q ack; then echo "Index settings updated..."; else echo "There was any issue updating the read-only attribute.  Please ensure Elasticsearch is running.";fi;

salt/common/tools/sbin/so-idh-restart

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# Copyright 2014-2022 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+. /usr/sbin/so-common
+
+/usr/sbin/so-restart idh $1

salt/common/tools/sbin/so-idh-start

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# Copyright 2014-2022 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+. /usr/sbin/so-common
+
+/usr/sbin/so-start idh $1

salt/common/tools/sbin/so-idh-stop

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# Copyright 2014-2022 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+. /usr/sbin/so-common
+
+/usr/sbin/so-stop idh $1

salt/common/tools/sbin/so-image-common

@@ -55,6 +55,7 @@ container_list() {
       "so-fleet"
       "so-fleet-launcher"
       "so-grafana"
+      "so-idh"
       "so-idstools"
       "so-influxdb"
       "so-kibana"
@@ -74,9 +75,6 @@ container_list() {
       "so-strelka-manager"
       "so-suricata"
       "so-telegraf"
-      "so-thehive"
-      "so-thehive-cortex"
-      "so-thehive-es"
       "so-wazuh"
       "so-zeek" 
     )

salt/common/tools/sbin/so-ip-update

@@ -53,7 +53,9 @@ if [ "$CONTINUE" == "y" ]; then
 	docker exec -i so-mysql mysql --user=root --password=$(lookup_pillar_secret 'mysql') -e "GRANT ALL PRIVILEGES ON *.* TO 'root'@'$NEW_IP' IDENTIFIED BY '$(lookup_pillar_secret 'mysql')' WITH GRANT OPTION;" &> /dev/null
 	echo "Removing MySQL root user from $OLD_IP"
 	docker exec -i so-mysql mysql --user=root --password=$(lookup_pillar_secret 'mysql') -e "DROP USER 'root'@'$OLD_IP';" &> /dev/null
-
+	echo "Updating Kibana dashboards"
+	salt-call state.apply kibana.so_savedobjects_defaults -l info queue=True
+	
 	echo "The IP has been changed from $OLD_IP to $NEW_IP."
 
 	echo

salt/common/tools/sbin/so-playbook-sigma-refresh

@@ -17,11 +17,21 @@
 
 . /usr/sbin/so-common
 
-# Regenerate ElastAlert & update Plays
-docker exec so-soctopus python3 playbook_play-update.py
+if ! [ -f /opt/so/state/playbook_regen_plays ] || [ "$1" = "--force" ]; then
 
-# Delete current Elastalert Rules
-rm /opt/so/rules/elastalert/playbook/*.yaml
+    echo "Refreshing Sigma & regenerating plays... "
 
-# Regenerate Elastalert Rules
-so-playbook-sync
+    # Regenerate ElastAlert & update Plays
+    docker exec so-soctopus python3 playbook_play-update.py
+
+    # Delete current Elastalert Rules
+    rm /opt/so/rules/elastalert/playbook/*.yaml
+
+    # Regenerate Elastalert Rules
+    so-playbook-sync
+    
+    # Create state file
+    touch /opt/so/state/playbook_regen_plays
+else
+  printf "\nState file found, exiting...\nRerun with --force to override.\n"
+fi

salt/common/tools/sbin/so-playbook-sync

@@ -18,7 +18,7 @@
 . /usr/sbin/so-common
 
 # Check to see if we are already running
-IS_RUNNING=$(ps aux | pgrep -f "so-playbook-sync" | wc -l)
-[ "$IS_RUNNING" -gt 3 ] && echo "$(date) - Multiple Playbook Sync processes already running...exiting." && exit 0
+NUM_RUNNING=$(pgrep -cf "/bin/bash /usr/sbin/so-playbook-sync")
+[ "$NUM_RUNNING" -gt 1 ] && echo "$(date) - $NUM_RUNNING Playbook sync processes running...exiting." && exit 0
 
 docker exec so-soctopus python3 playbook_play-sync.py

salt/common/tools/sbin/so-restart

@@ -15,7 +15,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-# Usage: so-restart  filebeat | kibana | playbook | thehive
+# Usage: so-restart  filebeat | kibana | playbook 
 
 . /usr/sbin/so-common
 
@@ -31,7 +31,6 @@ if [ $# -ge 1 ]; then
         fi
 
         case $1 in
-                "cortex") docker stop so-thehive-cortex so-thehive && docker rm so-thehive-cortex so-thehive && salt-call state.apply hive queue=True;;
                 "steno") docker stop so-steno && docker rm so-steno && salt-call state.apply pcap queue=True;;
                 *)  docker stop so-$1 ; docker rm so-$1 ; salt-call state.apply $1 queue=True;;
         esac

salt/common/tools/sbin/so-saltstack-update

@@ -32,11 +32,17 @@ copy_new_files() {
   # Copy new files over to the salt dir
   cd /tmp/sogh/securityonion
   git checkout $BRANCH
+  VERSION=$(cat VERSION)
+  # We need to overwrite if there is a repo file
+  if [ -d /opt/so/repo ]; then
+    tar -czf /opt/so/repo/"$VERSION".tar.gz -C "$(pwd)/.." .
+  fi
   rsync -a salt $default_salt_dir/
   rsync -a pillar $default_salt_dir/
   chown -R socore:socore $default_salt_dir/salt
   chown -R socore:socore $default_salt_dir/pillar
   chmod 755 $default_salt_dir/pillar/firewall/addfirewall.sh
+
   rm -rf /tmp/sogh
 }
 

salt/common/tools/sbin/so-sensor-clean

@@ -115,8 +115,8 @@ clean() {
 }
 
 # Check to see if we are already running
-IS_RUNNING=$(ps aux | pgrep -f "so-sensor-clean" | wc -l)
-[ "$IS_RUNNING" -gt 3 ] && echo "$(date) - $IS_RUNNING sensor clean script processes running...exiting." >>$LOG && exit 0
+NUM_RUNNING=$(pgrep -cf "/bin/bash /usr/sbin/so-sensor-clean")
+[ "$NUM_RUNNING" -gt 1 ] && echo "$(date) - $NUM_RUNNING sensor clean script processes running...exiting." >>$LOG && exit 0
 
 if [ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ]; then
 	while [ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ]; do

salt/common/tools/sbin/so-start

@@ -15,7 +15,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-# Usage: so-start  all | filebeat | kibana | playbook | thehive
+# Usage: so-start  all | filebeat | kibana | playbook 
 
 . /usr/sbin/so-common
 

salt/common/tools/sbin/so-status

@@ -15,10 +15,6 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-if ! [ "$(id -u)" = 0 ]; then
-   echo "This command must be run as root"
-   exit 1
-fi
 
 display_help() {
 cat <<HELP_USAGE
@@ -100,10 +96,15 @@ create_expected_container_list() {
 
 }
 
+# {% raw %}
 populate_container_lists() {
+    # TODO: check exit code directly, not with $?
     systemctl is-active --quiet docker
 
     if [[ $? = 0 ]]; then
+        # TODO: look into using docker templates instead of curl and jq
+        #  Ex docker ps --format "{{.Names}}\t{{.State}}"
+        # TODO: convert the output to an associtive array
         mapfile -t docker_raw_list < <(curl -s --unix-socket /var/run/docker.sock http:/v1.40/containers/json?all=1 \
             | jq -c '.[] | { Name: .Names[0], State: .State }' \
             | tr -d '/{"}')
@@ -167,60 +168,55 @@ parse_status() {
     fi
 }
 
-# {% raw %}
 
 print_line() {
-    local service_name=${1}
-    local service_state="$( parse_status ${1} ${2} )"
-    local columns=$(tput cols)
-    local state_color="\e[0m"
+    local service_name="${1}"
+    local service_state="" ; service_state="$( parse_status "${1}" "${2}" )"
+    # XXX: What will we do if tput isn't avalable?
+    local line=""
+    local PADDING_CONSTANT=""
+    local columns=35 # value used if not printing to a tty
 
-    local PADDING_CONSTANT=15
+    if (( __tty == 1 )); then
+        local reset_attr; reset_attr="$(tput sgr0)" # reset all attributes
+        local bold; bold="$(tput bold)"
+        local red; red="$(tput setaf 1)"
+        local green; green="$(tput setaf 2)"
+        local yellow; yellow="$(tput setaf 3)"
+        PADDING_CONSTANT=15 # whitespace + brackets + 1
 
-    if [[ $service_state = "$ERROR_STRING" ]] || [[ $service_state = "$MISSING_STRING" ]]; then
-        state_color="\e[1;31m"
-        if [[ "$EXITCODE" -eq 0 ]]; then
-            EXITCODE=1
-        fi
+        columns=$(tput cols)
+    fi
+
+    # construct a line of '------' so that the names and states are all aligned
+    linewidth=$(( columns - PADDING_CONSTANT - ${#service_name} - ${#service_state} ))
+    for i in $(seq 0 "${linewidth}"); do
+      line="${line}-"
+    done
+
+    if [[ $service_state = "$ERROR_STRING" ]] \
+        || [[ $service_state = "$MISSING_STRING" ]]; then
+          state_color="${red:-}"
+          if [[ "$EXITCODE" -eq 0 ]]; then
+              EXITCODE=1
+          fi
     elif [[ $service_state = "$SUCCESS_STRING" ]]; then
-        state_color="\e[1;32m"
-    elif [[ $service_state = "$PENDING_STRING" ]] || [[ $service_state = "$DISABLED_STRING" ]] || [[ $service_state = "$STARTING_STRING" ]] || [[ $service_state = "$WAIT_START_STRING" ]]; then
-        state_color="\e[1;33m"
-        EXITCODE=2
+        state_color="${green:-}"
+    elif [[ $service_state = "$PENDING_STRING" ]] \
+        || [[ $service_state = "$DISABLED_STRING" ]] \
+        || [[ $service_state = "$STARTING_STRING" ]] \
+        || [[ $service_state = "$WAIT_START_STRING" ]]; then
+            state_color="${yellow:-}"
+            EXITCODE=2
     fi
 
-    printf "    $service_name "
-    for i in $(seq 0 $(( $columns - $PADDING_CONSTANT - ${#service_name} - ${#service_state} ))); do
-        printf "${state_color}%b\e[0m" "-"
-    done
-    printf " [ "
-    printf "${state_color}%b\e[0m" "$service_state"
-    printf "%s    \n" " ]"
-}
-
-non_term_print_line() {
-    local service_name=${1}
-    local service_state="$( parse_status ${1} ${2} )"
-
-    if [[ $service_state = "$ERROR_STRING" ]] || [[ $service_state = "$MISSING_STRING" ]]; then
-        if [[ "$EXITCODE" -eq 0 ]]; then
-            EXITCODE=1
-        fi
-    elif [[ $service_state = "$PENDING_STRING" ]] || [[ $service_state = "$DISABLED_STRING" ]] || [[ $service_state = "$STARTING_STRING" ]] || [[ $service_state = "$WAIT_START_STRING" ]]; then
-        EXITCODE=2
-    fi
-
-    printf "    $service_name "
-    for i in $(seq 0 $(( 35 - ${#service_name} - ${#service_state} ))); do
-        printf "-"
-    done
-    printf " [ "
-    printf "$service_state"
-    printf "%s    \n" " ]"
+    service_state="${bold:-}${state_color:-}${service_state}${reset_attr:-}"
+    line="${bold:-}${state_color:-}${line:-}${reset_attr:-}"
+    printf "    %s %s [ %s ]    \n" "${service_name}" "${line:-}" "${service_state}"
 }
 
 main() {
-
+    is_tty
     # if running from salt
     if [ "$CALLER" == 'salt-call' ] ||  [ "$CALLER" == 'salt-minion' ]; then
       printf "\n"
@@ -228,20 +224,19 @@ main() {
 
       systemctl is-active --quiet docker
       if [[ $? = 0 ]]; then
-          non_term_print_line "Docker" "running"
+          print_line "Docker" "running"
       else
-          non_term_print_line "Docker" "exited"
+          print_line "Docker" "exited"
       fi
 
       populate_container_lists
 
-      printf "\n"
-      printf "Checking container statuses\n\n"
+      printf "\nChecking container statuses\n\n"
 
       local num_containers=${#container_name_list[@]}
 
       for i in $(seq 0 $(($num_containers - 1 ))); do
-          non_term_print_line ${container_name_list[$i]} ${container_state_list[$i]}
+          print_line ${container_name_list[$i]} ${container_state_list[$i]}
       done
 
       printf "\n"
@@ -257,9 +252,12 @@ main() {
       else
           print_or_parse="print_line"
 
-          local focus_color="\e[1;34m"
-          printf "\n"
-          printf "${focus_color}%b\e[0m" "Checking Docker status\n\n"
+          if (( __tty == 1 )) ; then
+          local bold; bold="$(tput bold)"
+          local focus_color; focus_color="$(tput setaf 4)"
+          local reset_attr; reset_attr="$(tput sgr0)" # reset all attributes
+          fi
+          printf "\n${bold}${focus_color:-}%s${reset_attr:-}\n\n" "Checking Docker status"
       fi
 
       systemctl is-active --quiet docker
@@ -272,8 +270,7 @@ main() {
       populate_container_lists
 
       if [ "$QUIET" = false ]; then
-          printf "\n"
-          printf "${focus_color}%b\e[0m" "Checking container statuses\n\n"
+          printf "\n${bold}${focus_color:-}%s${reset_attr:-}\n\n" "Checking container statuses"
       fi
 
       local num_containers=${#container_name_list[@]}
@@ -288,20 +285,30 @@ main() {
     fi
 }
 
+is_tty() {
+    __tty=0
+    [ -t 1 ] && __tty=1
+    # don't print colors if NO_COLOR is set to anything
+    [ "${#NO_COLOR}" -ne 0 ] && __tty=0
+}
+
 # {% endraw %}
 
+if ! [ "$(id -u)" = 0 ]; then
+   echo "${0}: This command must be run as root"
+   exit 1
+fi
+
 while getopts ':hq' OPTION; do
   case "$OPTION" in
     h)
       display_help
       exit 0
       ;;
-    q)
-      QUIET=true
-      ;;
+    q) QUIET=true ;;
     \?)
       display_help
-      exit 0
+      exit 1
       ;;
   esac
 done

salt/common/tools/sbin/so-thehive-es-restart

@@ -17,5 +17,4 @@
 
 . /usr/sbin/so-common
 
-/usr/sbin/so-stop thehive-es $1
-/usr/sbin/so-start thehive $1
+echo "TheHive and its components are no longer part of Security Onion"

salt/common/tools/sbin/so-thehive-es-start

@@ -17,4 +17,4 @@
 
 . /usr/sbin/so-common
 
-/usr/sbin/so-start thehive $1
+echo "TheHive and its components are no longer part of Security Onion"

salt/common/tools/sbin/so-thehive-es-stop

@@ -17,4 +17,4 @@
 
 . /usr/sbin/so-common
 
-/usr/sbin/so-stop thehive-es $1
+echo "TheHive and its components are no longer part of Security Onion"

salt/common/tools/sbin/so-thehive-restart

@@ -17,4 +17,4 @@
 
 . /usr/sbin/so-common
 
-/usr/sbin/so-restart thehive $1
+echo "TheHive and its components are no longer part of Security Onion"

salt/common/tools/sbin/so-thehive-start

@@ -17,4 +17,4 @@
 
 . /usr/sbin/so-common
 
-/usr/sbin/so-start thehive $1
+echo "TheHive and its components are no longer part of Security Onion"

salt/common/tools/sbin/so-thehive-stop

@@ -17,4 +17,4 @@
 
 . /usr/sbin/so-common
 
-/usr/sbin/so-stop thehive $1
+echo "TheHive and its components are no longer part of Security Onion"

salt/common/tools/sbin/so-thehive-user-add

@@ -17,38 +17,4 @@
 
 . /usr/sbin/so-common
 
-usage() {
-    echo "Usage: $0 <new-user-name>"
-    echo ""
-    echo "Adds a new user to TheHive. The new password will be read from STDIN."
-    exit 1
-}
-
-if [ $# -ne 1 ]; then
-  usage
-fi
-
-USER=$1
-
-THEHIVE_KEY=$(lookup_pillar hivekey)
-THEHVIE_API_URL="$(lookup_pillar url_base)/thehive/api"
-THEHIVE_USER=$USER
-
-# Read password for new user from stdin
-test -t 0
-if [[ $? == 0 ]]; then
-  echo "Enter new password:"
-fi
-read -rs THEHIVE_PASS
-
-check_password_and_exit "$THEHIVE_PASS"
-
-# Create new user in TheHive
-resp=$(curl -sk -XPOST -H "Authorization: Bearer $THEHIVE_KEY" -H "Content-Type: application/json" -L "https://$THEHVIE_API_URL/user" -d "{\"login\" : \"$THEHIVE_USER\",\"name\" : \"$THEHIVE_USER\",\"roles\" : [\"read\",\"alert\",\"write\",\"admin\"],\"preferences\" : \"{}\",\"password\" : \"$THEHIVE_PASS\"}")
-if [[ "$resp" =~ \"status\":\"Ok\" ]]; then
-    echo "Successfully added user to TheHive"
-else
-    echo "Unable to add user to TheHive; user might already exist"
-    echo $resp
-    exit 2
-fi
+echo "TheHive and its components are no longer part of Security Onion"

salt/common/tools/sbin/so-thehive-user-enable

@@ -17,41 +17,4 @@
 
 . /usr/sbin/so-common
 
-usage() {
-    echo "Usage: $0 <user-name> <true|false>"
-    echo ""
-    echo "Enables or disables a user in TheHive."
-    exit 1
-}
-
-if [ $# -ne 2 ]; then
-  usage
-fi
-
-USER=$1
-
-THEHIVE_KEY=$(lookup_pillar hivekey)
-THEHVIE_API_URL="$(lookup_pillar url_base)/thehive/api"
-THEHIVE_USER=$USER
-
-case "${2^^}" in
-	FALSE | NO | 0)
-		THEHIVE_STATUS=Locked
-		;;
-	TRUE | YES | 1)
-		THEHIVE_STATUS=Ok
-		;;
-	*)
-		usage
-		;;
-esac
-
-resp=$(curl -sk -XPATCH -H "Authorization: Bearer $THEHIVE_KEY" -H "Content-Type: application/json" -L "https://$THEHVIE_API_URL/user/${THEHIVE_USER}" -d "{\"status\":\"${THEHIVE_STATUS}\" }")
-if [[ "$resp" =~ \"status\":\"Locked\" || "$resp" =~ \"status\":\"Ok\" ]]; then
-    echo "Successfully updated user in TheHive"
-else
-    echo "Failed to update user in TheHive"
-    echo "$resp"
-    exit 2
-fi
-    
+echo "TheHive and its components are no longer part of Security Onion"

salt/common/tools/sbin/so-thehive-user-update

@@ -17,41 +17,4 @@
 
 . /usr/sbin/so-common
 
-usage() {
-    echo "Usage: $0 <user-name>"
-    echo ""
-    echo "Update password for an existing TheHive user. The new password will be read from STDIN."
-    exit 1
-}
-
-if [ $# -ne 1 ]; then
-  usage
-fi
-
-USER=$1
-
-THEHIVE_KEY=$(lookup_pillar hivekey)
-THEHVIE_API_URL="$(lookup_pillar url_base)/thehive/api"
-THEHIVE_USER=$USER
-
-# Read password for new user from stdin
-test -t 0
-if [[ $? == 0 ]]; then
-  echo "Enter new password:"
-fi
-read -rs THEHIVE_PASS
-
-if ! check_password "$THEHIVE_PASS"; then
-  echo "Password is invalid. Please exclude single quotes, double quotes, dollar signs, and backslashes from the password."
-  exit 2
-fi
-
-# Change password for user in TheHive
-resp=$(curl -sk -XPOST -H "Authorization: Bearer $THEHIVE_KEY" -H "Content-Type: application/json" -L "https://$THEHVIE_API_URL/user/${THEHIVE_USER}/password/set" -d "{\"password\" : \"$THEHIVE_PASS\"}")
-if [[ -z "$resp" ]]; then
-    echo "Successfully updated TheHive user password"
-else
-    echo "Unable to update TheHive user password"
-    echo $resp
-    exit 2
-fi
+echo "TheHive and its components are no longer part of Security Onion"

salt/common/tools/sbin/so-user

@@ -29,7 +29,7 @@ if [[ $# -lt 1 || $# -gt 3 ]]; then
   echo "      add: Adds a new user to the identity system; requires 'email' parameter, while 'role' parameter is optional and defaults to $DEFAULT_ROLE"
   echo "  addrole: Grants a role to an existing user; requires 'email' and 'role' parameters"
   echo "  delrole: Removes a role from an existing user; requires 'email' and 'role' parameters"
-  echo "   update: Updates a user's password; requires 'email' parameter"
+  echo "   update: Updates a user's password and disables MFA; requires 'email' parameter"
   echo "   enable: Enables a user; requires 'email' parameter"
   echo "  disable: Disables a user; requires 'email' parameter"
   echo " validate: Validates that the given email address and password are acceptable; requires 'email' parameter"
@@ -44,8 +44,9 @@ operation=$1
 email=$2
 role=$3
 
-kratosUrl=${KRATOS_URL:-http://127.0.0.1:4434}
+kratosUrl=${KRATOS_URL:-http://127.0.0.1:4434/admin}
 databasePath=${KRATOS_DB_PATH:-/opt/so/conf/kratos/db/db.sqlite}
+databaseTimeout=${KRATOS_DB_TIMEOUT:-5000}
 bcryptRounds=${BCRYPT_ROUNDS:-12}
 elasticUsersFile=${ELASTIC_USERS_FILE:-/opt/so/saltstack/local/salt/elasticsearch/files/users}
 elasticRolesFile=${ELASTIC_ROLES_FILE:-/opt/so/saltstack/local/salt/elasticsearch/files/users_roles}
@@ -98,7 +99,7 @@ function validatePassword() {
   password=$1
 
   len=$(expr length "$password")
-  if [[ $len -lt 6 ]]; then
+  if [[ $len -lt 8 ]]; then
     fail "Password does not meet the minimum requirements"
   fi
   if [[ $len -gt 72 ]]; then
@@ -147,7 +148,10 @@ function updatePassword() {
     # Generate password hash
     passwordHash=$(hashPassword "$password")
     # Update DB with new hash
-    echo "update identity_credentials set config=CAST('{\"hashed_password\":\"$passwordHash\"}' as BLOB), updated_at=datetime('now') where identity_id='${identityId}';" | sqlite3 "$databasePath"
+    echo "update identity_credentials set config=CAST('{\"hashed_password\":\"$passwordHash\"}' as BLOB), updated_at=datetime('now') where identity_id='${identityId}' and identity_credential_type_id=(select id from identity_credential_types where name='password');" | sqlite3 -cmd ".timeout ${databaseTimeout}" "$databasePath"
+    # Deactivate MFA
+    echo "delete from identity_credential_identifiers where identity_credential_id=(select id from identity_credentials where identity_id='${identityId}' and identity_credential_type_id=(select id from identity_credential_types where name='totp'));" | sqlite3 -cmd ".timeout ${databaseTimeout}" "$databasePath"
+    echo "delete from identity_credentials where identity_id='${identityId}' and identity_credential_type_id=(select id from identity_credential_types where name='totp');" | sqlite3 -cmd ".timeout ${databaseTimeout}" "$databasePath"
     [[ $? != 0 ]] && fail "Unable to update password"
   fi
 }
@@ -172,7 +176,7 @@ function ensureRoleFileExists() {
     if [[ -f "$databasePath" ]]; then
       echo "Migrating roles to new file: $socRolesFile"
 
-      echo "select 'superuser:' || id from identities;" | sqlite3 "$databasePath" \
+      echo "select 'superuser:' || id from identities;" | sqlite3 -cmd ".timeout ${databaseTimeout}" "$databasePath" \
         >> "$rolesTmpFile"
       [[ $? != 0 ]] && fail "Unable to read identities from database"
 
@@ -243,27 +247,34 @@ function syncElastic() {
 
   if [[ -f "$databasePath" && -f "$socRolesFile" ]]; then
     # Append the SOC users 
-    echo "select '{\"user\":\"' || ici.identifier || '\", \"data\":' || ic.config || '}'" \
-      "from identity_credential_identifiers ici, identity_credentials ic, identities i " \
+    userData=$(echo "select '{\"user\":\"' || ici.identifier || '\", \"data\":' || ic.config || '}'" \
+      "from identity_credential_identifiers ici, identity_credentials ic, identities i, identity_credential_types ict " \
       "where " \
       "      ici.identity_credential_id=ic.id " \
       "  and ic.identity_id=i.id " \
+      "  and ict.id=ic.identity_credential_type_id " \
+      "  and ict.name='password' " \
       "  and instr(ic.config, 'hashed_password') " \
       "  and i.state == 'active' " \
       "order by ici.identifier;" | \
-      sqlite3 "$databasePath" | \
+      sqlite3 -cmd ".timeout ${databaseTimeout}" "$databasePath")
+    [[ $? != 0 ]] && fail "Unable to read credential hashes from database"
+    echo "${userData}" | \
       jq -r '.user + ":" + .data.hashed_password' \
       >> "$usersTmpFile"
-    [[ $? != 0 ]] && fail "Unable to read credential hashes from database"
 
     # Append the user roles
     while IFS="" read -r rolePair || [ -n "$rolePair" ]; do
       userId=$(echo "$rolePair" | cut -d: -f2)
       role=$(echo "$rolePair" | cut -d: -f1)
       echo "select '$role:' || ici.identifier " \
-        "from identity_credential_identifiers ici, identity_credentials ic " \
-        "where ici.identity_credential_id=ic.id and ic.identity_id = '$userId';" | \
-        sqlite3 "$databasePath" >> "$rolesTmpFile"
+        "from identity_credential_identifiers ici, identity_credentials ic, identity_credential_types ict " \
+        "where ici.identity_credential_id=ic.id " \
+        "  and ict.id=ic.identity_credential_type_id " \
+        "  and ict.name='password' " \
+        "  and ic.identity_id = '$userId';" | \
+        sqlite3 -cmd ".timeout ${databaseTimeout}" "$databasePath" >> "$rolesTmpFile"
+      [[ $? != 0 ]] && fail "Unable to read role identities from database"
     done < "$socRolesFile"
 
   else
@@ -293,7 +304,8 @@ function syncAll() {
   if [[ -z "$FORCE_SYNC" && -f "$databasePath" && -f "$elasticUsersFile" ]]; then
     usersFileAgeSecs=$(echo $(($(date +%s) - $(date +%s -r "$elasticUsersFile"))))
     staleCount=$(echo "select count(*) from identity_credentials where updated_at >= Datetime('now', '-${usersFileAgeSecs} seconds');" \
-      | sqlite3 "$databasePath")
+      | sqlite3 -cmd ".timeout ${databaseTimeout}" "$databasePath")
+    [[ $? != 0 ]] && fail "Unable to read user count from database"
     if [[ "$staleCount" == "0" && "$elasticRolesFile" -nt "$socRolesFile" ]]; then
       return 1
     fi
@@ -396,7 +408,7 @@ function migrateLockedUsers() {
   # This is a migration function to convert locked users from prior to 2.3.90
   # to inactive users using the newer Kratos functionality. This should only
   # find locked users once.
-  lockedEmails=$(curl -s http://localhost:4434/identities | jq -r '.[] | select(.traits.status == "locked") | .traits.email')
+  lockedEmails=$(curl -s ${kratosUrl}/identities | jq -r '.[] | select(.traits.status == "locked") | .traits.email')
   if [[ -n "$lockedEmails" ]]; then
     echo "Disabling locked users..."
     for email in $lockedEmails; do
@@ -464,7 +476,6 @@ case "${operation}" in
     createUser "$email" "${role:-$DEFAULT_ROLE}"
     syncAll
     echo "Successfully added new user to SOC"
-    check_container thehive && echo "$password" | so-thehive-user-add "$email"
     check_container fleet && echo "$password" | so-fleet-user-add "$email"
     ;;
 
@@ -516,7 +527,6 @@ case "${operation}" in
     updateStatus "$email" 'active'
     syncAll
     echo "Successfully enabled user"
-    check_container thehive && so-thehive-user-enable "$email" true
     echo "Fleet user will need to be recreated manually with so-fleet-user-add"
     ;;
 
@@ -528,7 +538,6 @@ case "${operation}" in
     updateStatus "$email" 'locked'
     syncAll
     echo "Successfully disabled user"
-    check_container thehive && so-thehive-user-enable "$email" false
     check_container fleet && so-fleet-user-delete "$email"   
     ;;    
 
@@ -540,7 +549,6 @@ case "${operation}" in
     deleteUser "$email"
     syncAll
     echo "Successfully deleted user"
-    check_container thehive && so-thehive-user-enable "$email" false
     check_container fleet && so-fleet-user-delete "$email"
     ;;
 

salt/common/tools/sbin/soup

@@ -34,7 +34,15 @@ check_err() {
   local err_msg="Unhandled error occured, please check $SOUP_LOG for details."
 
   [[ $ERR_HANDLED == true ]] && exit $exit_code
+
   if [[ $exit_code -ne 0 ]]; then
+  
+    set +e
+    systemctl_func "start" "$cron_service_name"
+    systemctl_func "start" "salt-master"
+    systemctl_func "start" "salt-minion"
+    enable_highstate
+
     printf '%s' "Soup failed with error $exit_code: "
     case $exit_code in
       2)
@@ -91,10 +99,7 @@ check_err() {
     if [[ $exit_code -ge 64 && $exit_code -le 113 ]]; then
       echo "$err_msg"
     fi
-    set +e
-    systemctl_func "start" "$cron_service_name"
-    echo "Ensuring highstate is enabled."
-    salt-call state.enable highstate --local
+
     exit $exit_code
   fi
 
@@ -158,7 +163,7 @@ EOF
 }
 
 airgap_update_dockers() {
-  if [[ $is_airgap -eq 0 ]]; then
+  if [[ $is_airgap -eq 0 ]] || [[ ! -z "$ISOLOC" ]]; then
     # Let's copy the tarball
     if [[ ! -f $AGDOCKER/registry.tar ]]; then
       echo "Unable to locate registry. Exiting"
@@ -245,7 +250,6 @@ check_sudoers() {
 }
 
 check_log_size_limit() {
-
   local num_minion_pillars
   num_minion_pillars=$(find /opt/so/saltstack/local/pillar/minions/ -type f | wc -l)
   
@@ -255,7 +259,7 @@ check_log_size_limit() {
     fi
   else
     local minion_id
-    minion_id=$(lookup_salt_value "id" "" "grains")
+    minion_id=$(lookup_salt_value "id" "" "grains" "" "local")
 
     local minion_arr
     IFS='_' read -ra minion_arr <<< "$minion_id"
@@ -263,7 +267,15 @@ check_log_size_limit() {
     local node_type="${minion_arr[0]}"
     
     local current_limit
-    current_limit=$(lookup_pillar "log_size_limit" "elasticsearch")
+    # since it is possible for the salt-master service to be stopped when this is run, we need to check the pillar values locally
+    # we need to combine default local and default pillars before doing this so we can define --pillar-root in salt-call
+    local epoch_date=$(date +%s%N)
+    mkdir -vp /opt/so/saltstack/soup_tmp_${epoch_date}/
+    cp -r /opt/so/saltstack/default/pillar/ /opt/so/saltstack/soup_tmp_${epoch_date}/
+    # use \cp here to overwrite any pillar files from default with those in local for the tmp directory
+    \cp -r /opt/so/saltstack/local/pillar/ /opt/so/saltstack/soup_tmp_${epoch_date}/
+    current_limit=$(salt-call pillar.get elasticsearch:log_size_limit --local --pillar-root=/opt/so/saltstack/soup_tmp_${epoch_date}/pillar --out=newline_values_only)
+    rm -rf /opt/so/saltstack/soup_tmp_${epoch_date}/
     
     local percent
     case $node_type in
@@ -359,6 +371,12 @@ clone_to_tmp() {
   fi
 }
 
+enable_highstate() {
+    echo "Enabling highstate."
+    salt-call state.enable highstate -l info --local
+    echo ""
+}
+
 generate_and_clean_tarballs() {
   local new_version
   new_version=$(cat $UPDATE_DIR/VERSION)
@@ -403,6 +421,9 @@ preupgrade_changes() {
     [[ "$INSTALLEDVERSION" == 2.3.50 || "$INSTALLEDVERSION" == 2.3.51 || "$INSTALLEDVERSION" == 2.3.52 || "$INSTALLEDVERSION" == 2.3.60 || "$INSTALLEDVERSION" == 2.3.61 || "$INSTALLEDVERSION" == 2.3.70 ]] && up_to_2.3.80
     [[ "$INSTALLEDVERSION" == 2.3.80 ]] && up_to_2.3.90
     [[ "$INSTALLEDVERSION" == 2.3.90 || "$INSTALLEDVERSION" == 2.3.91 ]] && up_to_2.3.100
+    [[ "$INSTALLEDVERSION" == 2.3.100 ]] && up_to_2.3.110
+    [[ "$INSTALLEDVERISON" == 2.3.110 ]] && up_to_2.3.120
+    [[ "$INSTALLEDVERISON" == 2.3.120 ]] && up_to_2.3.130
     true
 }
 
@@ -415,6 +436,11 @@ postupgrade_changes() {
     [[ "$POSTVERSION" == 2.3.40 || "$POSTVERSION" == 2.3.50 || "$POSTVERSION" == 2.3.51 || "$POSTVERSION" == 2.3.52 ]] && post_to_2.3.60
     [[ "$POSTVERSION" == 2.3.60 || "$POSTVERSION" == 2.3.61 || "$POSTVERSION" == 2.3.70 || "$POSTVERSION" == 2.3.80 ]] && post_to_2.3.90
     [[ "$POSTVERSION" == 2.3.90 || "$POSTVERSION" == 2.3.91 ]] && post_to_2.3.100
+    [[ "$POSTVERSION" == 2.3.100 ]] && post_to_2.3.110
+    [[ "$POSTVERSION" == 2.3.110 ]] && post_to_2.3.120
+    [[ "$POSTVERSION" == 2.3.120 ]] && post_to_2.3.130
+    
+
     true
 }
 
@@ -466,19 +492,40 @@ post_to_2.3.90() {
 
 post_to_2.3.100() {
   echo "Post Processing for 2.3.100"
+  POSTVERSION=2.3.100
+}
+
+post_to_2.3.110() {
+  echo "Post Processing for 2.3.110"
+  echo "Removing old Elasticsearch index templates"
+  [ -d /opt/so/saltstack/default/salt/elasticsearch/templates/so ] && rm -rf /opt/so/saltstack/default/salt/elasticsearch/templates/so
   echo "Updating Kibana dashboards"
   salt-call state.apply kibana.so_savedobjects_defaults queue=True
+  POSTVERSION=2.3.110
 }
 
+post_to_2.3.120() {
+  echo "Post Processing for 2.3.120"
+  POSTVERSION=2.3.120
+  sed -i '/so-thehive-es/d;/so-thehive/d;/so-cortex/d' /opt/so/conf/so-status/so-status.conf
+}
+
+post_to_2.3.130() {
+  echo "Post Processing for 2.3.130"
+  POSTVERSION=2.3.130
+}
+
+
+
 stop_salt_master() {
     # kill all salt jobs across the grid because the hang indefinitely if they are queued and salt-master restarts
     set +e
     echo ""
     echo "Killing all Salt jobs across the grid."
-    salt \* saltutil.kill_all_jobs
+    salt \* saltutil.kill_all_jobs >> $SOUP_LOG 2>&1
     echo ""
     echo "Killing any queued Salt jobs on the manager."
-    pkill -9 -ef "/usr/bin/python3 /bin/salt"
+    pkill -9 -ef "/usr/bin/python3 /bin/salt" >> $SOUP_LOG 2>&1
     set -e
 
     echo ""
@@ -704,12 +751,8 @@ up_to_2.3.90() {
 }
 
 up_to_2.3.100() {
-  echo "Updating to Security Onion 2.3.100"
   fix_wazuh
   
-  echo "Removing /opt/so/state files for patched Salt InfluxDB module and state. This is due to Salt being upgraded and needing to patch the files again."
-  rm -vrf /opt/so/state/influxdb_continuous_query.py.patched /opt/so/state/influxdb_retention_policy.py.patched /opt/so/state/influxdbmod.py.patched
-  
   echo "Adding receiver hostgroup with so-firewall"
   if so-firewall addhostgroup receiver 2>&1 | grep -q 'Already exists'; then
     echo 'receiver hostgroup already exists'
@@ -721,6 +764,22 @@ up_to_2.3.100() {
   grep -qxF "  receiver:" /opt/so/saltstack/local/salt/firewall/assigned_hostgroups.local.map.yaml || sed -i -e '$a\  receiver:' /opt/so/saltstack/local/salt/firewall/assigned_hostgroups.local.map.yaml
 }
 
+up_to_2.3.110() {
+  sed -i 's|shards|index_template:\n        template:\n          settings:\n            index:\n              number_of_shards|g' /opt/so/saltstack/local/pillar/global.sls
+}
+
+up_to_2.3.120() {
+  # Stop thehive services since these will be broken in .120
+  so-thehive-stop
+  so-thehive-es-stop
+  so-cortex-stop
+}
+
+up_to_2.3.130() {
+  # Remove file for nav update
+  rm -f /opt/so/conf/navigator/layers/nav_layer_playbook.json
+}
+
 verify_upgradespace() {
   CURRENTSPACE=$(df -BG / | grep -v Avail | awk '{print $4}' | sed 's/.$//')
   if [ "$CURRENTSPACE" -lt "10" ]; then
@@ -743,29 +802,6 @@ upgrade_space() {
   fi  
 }
 
-thehive_maint() {
-  echo -n "Waiting for TheHive..."
-  COUNT=0
-  THEHIVE_CONNECTED="no"
-  while [[ "$COUNT" -le 240 ]]; do
-    curl --output /dev/null --silent --head --fail -k "https://localhost/thehive/api/alert"
-      if [ $? -eq 0 ]; then
-        THEHIVE_CONNECTED="yes"
-        echo "connected!"
-        break
-      else
-        ((COUNT+=1))
-        sleep 1
-        echo -n "."
-      fi
-  done
-  if [ "$THEHIVE_CONNECTED" == "yes" ]; then
-    echo "Migrating thehive databases if needed."
-    curl -v -k -XPOST -L "https://localhost/thehive/api/maintenance/migrate" >> "$SOUP_LOG" 2>&1
-    curl -v -k -XPOST -L "https://localhost/cortex/api/maintenance/migrate" >> "$SOUP_LOG" 2>&1
-  fi
-}
-
 unmount_update() {
   cd /tmp
   umount /tmp/soagupdate
@@ -858,7 +894,7 @@ upgrade_salt() {
     echo ""
     set +e
     run_check_net_err \
-    "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -X -F -M -x python3 stable \"$NEWSALTVERSION\"" \
+    "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -X -r -F -M -x python3 stable \"$NEWSALTVERSION\"" \
     "Could not update salt, please check $SOUP_LOG for details."
     set -e
     echo "Applying apt hold for Salt."
@@ -867,11 +903,29 @@ upgrade_salt() {
     apt-mark hold "salt-master"
     apt-mark hold "salt-minion"
   fi
+
+  echo "Checking if Salt was upgraded."
+  echo ""
+  # Check that Salt was upgraded
+  SALTVERSIONPOSTUPGRADE=$(salt --versions-report | grep Salt: | awk '{print $2}')
+  if [[ "$SALTVERSIONPOSTUPGRADE" != "$NEWSALTVERSION" ]]; then
+    echo "Salt upgrade failed. Check of indicators of failure in $SOUP_LOG."
+    echo "Once the issue is resolved, run soup again."
+    echo "Exiting."
+    echo ""
+    exit 0
+  else
+    echo "Salt upgrade success."
+    echo ""
+    echo "Removing /opt/so/state files for patched Salt InfluxDB module and state. This is due to Salt being upgraded and needing to patch the files again."
+    rm -vrf /opt/so/state/influxdb_continuous_query.py.patched /opt/so/state/influxdb_retention_policy.py.patched /opt/so/state/influxdbmod.py.patched
+  fi
+
 }
 
 update_repo() {
-  echo "Performing repo changes."
   if [[ "$OS" == "centos" ]]; then
+    echo "Performing repo changes."
     # Import GPG Keys
     gpg_rpm_import
     echo "Disabling fastestmirror."
@@ -891,6 +945,21 @@ update_repo() {
       yum clean all
       yum repolist
     fi
+  elif [[ "$OS" == "ubuntu" ]]; then
+    ubuntu_version=$(grep VERSION_ID /etc/os-release | awk -F '[ "]' '{print $2}')
+
+    if grep -q "UBUNTU_CODENAME=bionic" /etc/os-release; then
+      OSVER=bionic
+    elif grep -q "UBUNTU_CODENAME=focal" /etc/os-release; then
+      OSVER=focal
+    else
+      echo "We do not support your current version of Ubuntu."
+      exit 1
+    fi
+
+    rm -f /etc/apt/sources.list.d/salt.list
+    echo "deb https://repo.securityonion.net/file/securityonion-repo/ubuntu/$ubuntu_version/amd64/salt $OSVER main" > /etc/apt/sources.list.d/saltstack.list
+    apt-get update
   fi
 }
 
@@ -923,6 +992,8 @@ verify_latest_update_script() {
 apply_hotfix() {
   if [[ "$INSTALLEDVERSION" == "2.3.90" ]] ; then
     fix_wazuh
+  elif [[ "$INSTALLEDVERSION" == "2.3.110" ]] ; then
+    2_3_10_hotfix_1
   else
     echo "No actions required. ($INSTALLEDVERSION/$HOTFIXVERSION)"
   fi
@@ -944,6 +1015,28 @@ fix_wazuh() {
   fi
 }
 
+#upgrade salt to 3004.1
+2_3_10_hotfix_1() {
+    systemctl_func "stop" "$cron_service_name"
+    # update mine items prior to stopping salt-minion and salt-master
+    update_salt_mine
+    stop_salt_minion
+    stop_salt_master
+    update_repo
+    # Does salt need upgraded. If so update it.
+    if [[ $UPGRADESALT -eq 1 ]]; then
+      echo "Upgrading Salt"
+      # Update the repo files so it can actually upgrade
+      upgrade_salt
+    fi
+    rm -f /opt/so/state/influxdb_continuous_query.py.patched /opt/so/state/influxdbmod.py.patched /opt/so/state/influxdb_retention_policy.py.patched
+    systemctl_func "start" "salt-master"
+    salt-call state.apply salt.python3-influxdb -l info
+    systemctl_func "start" "salt-minion"
+    systemctl_func "start" "$cron_service_name"
+
+}
+
 main() {
   trap 'check_err $?' EXIT
   
@@ -955,6 +1048,17 @@ main() {
   echo "### Preparing soup at $(date) ###"
   echo ""
 
+  set_os
+  set_cron_service_name
+  if ! check_salt_master_status; then
+  	echo "Could not talk to salt master"
+    echo "Please run 'systemctl status salt-master' to ensure the salt-master service is running and check the log at /opt/so/log/salt/master."
+    echo "SOUP will now attempt to start the salt-master service and exit."
+    exit 1
+  fi
+
+  echo "This node can communicate with the salt-master."
+
   echo "Checking to see if this is a manager."
   echo ""
   require_manager
@@ -976,6 +1080,11 @@ main() {
     # Let's mount the ISO since this is airgap
     airgap_mounted
   else
+    # if not airgap but -f was used
+    if [[ ! -z "$ISOLOC" ]]; then
+      airgap_mounted
+      AGDOCKER=/tmp/soagupdate/docker
+    fi
     echo "Cloning Security Onion github repo into $UPDATE_DIR."
     echo "Removing previous upgrade sources."
     rm -rf $UPDATE_DIR
@@ -985,8 +1094,6 @@ main() {
   echo "Verifying we have the latest soup script."
   verify_latest_update_script
   echo ""
-  set_os
-  set_cron_service_name
   set_palette
   check_elastic_license
   echo ""
@@ -1008,12 +1115,19 @@ main() {
   upgrade_check_salt
   set -e
 
+  if [[ $is_airgap -eq 0 ]]; then
+    update_centos_repo
+    yum clean all
+    check_os_updates
+  fi
+
   if [ "$is_hotfix" == "true" ]; then
     echo "Applying $HOTFIXVERSION hotfix"
     copy_new_files
     apply_hotfix
     echo "Hotfix applied"
     update_version
+    enable_highstate
     salt-call state.highstate -l info queue=True
   else
     echo ""
@@ -1028,9 +1142,10 @@ main() {
     echo "Updating dockers to $NEWVERSION."
     if [[ $is_airgap -eq 0 ]]; then
       airgap_update_dockers
-      update_centos_repo
-      yum clean all
-      check_os_updates
+    # if not airgap but -f was used
+    elif [[ ! -z "$ISOLOC" ]]; then
+      airgap_update_dockers
+      unmount_update
     else
       update_registry
       set +e
@@ -1049,21 +1164,6 @@ main() {
       echo "Upgrading Salt"
       # Update the repo files so it can actually upgrade
       upgrade_salt
-    
-      echo "Checking if Salt was upgraded."
-      echo ""
-      # Check that Salt was upgraded
-      SALTVERSIONPOSTUPGRADE=$(salt --versions-report | grep Salt: | awk '{print $2}')
-      if [[ "$SALTVERSIONPOSTUPGRADE" != "$NEWSALTVERSION" ]]; then
-        echo "Salt upgrade failed. Check of indicators of failure in $SOUP_LOG."
-        echo "Once the issue is resolved, run soup again."
-        echo "Exiting."
-        echo ""
-        exit 0
-      else
-        echo "Salt upgrade success."
-        echo ""
-      fi
     fi
 
     preupgrade_changes
@@ -1119,9 +1219,7 @@ main() {
       echo ""
     fi
 
-    echo "Enabling highstate."
-    salt-call state.enable highstate -l info --local
-    echo ""
+    enable_highstate
 
     echo ""
     echo "Running a highstate. This could take several minutes."
@@ -1144,7 +1242,6 @@ main() {
     salt-call state.highstate -l info queue=True
     postupgrade_changes
     [[ $is_airgap -eq 0 ]] && unmount_update
-    thehive_maint
 
     echo ""
     echo "Upgrade to $NEWVERSION complete."

salt/curator/files/action/delete.yml

@@ -18,6 +18,10 @@ actions:
     - filtertype: pattern
       kind: regex
       value: '^(logstash-.*|so-.*)$'
+    - filtertype: pattern
+      kind: regex
+      value: '^(so-case.*)$'
+      exclude: True
     - filtertype: space
       source: creation_date
       use_age: True

salt/curator/files/bin/so-curator-closed-delete-delete

@@ -34,9 +34,13 @@ overlimit() {
 
 closedindices() {
 
-        INDICES=$({{ ELASTICCURL }} -s -k https://{{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/_cat/indices?h=index\&expand_wildcards=closed 2> /dev/null)
+	# If we can't query Elasticsearch, then immediately return false.
+        {{ ELASTICCURL }} -s -k https://{{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/_cat/indices?h=index\&expand_wildcards=closed >/dev/null 2>&1
         [ $? -eq 1 ] && return false 
-        echo ${INDICES} | grep -q -E "(logstash-|so-)"
+	# First, get the list of closed indices using _cat/indices?h=index\&expand_wildcards=closed.
+	# Next, filter out any so-case indices.
+	# Finally, use grep's -q option to return true if there are any remaining logstash- or so- indices.
+        {{ ELASTICCURL }} -s -k https://{{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/_cat/indices?h=index\&expand_wildcards=closed | grep -v "so-case" | grep -q -E "(logstash-|so-)"
 }
 
 # Check for 2 conditions:
@@ -47,9 +51,10 @@ while overlimit && closedindices; do
 
 	# We need to determine OLDEST_INDEX:
 	# First, get the list of closed indices using _cat/indices?h=index\&expand_wildcards=closed.
-	# Then, sort by date by telling sort to use hyphen as delimiter and then sort on the third field.
+	# Next, filter out any so-case indices and only select the remaining logstash- or so- indices.
+	# Then, sort by date by telling sort to use hyphen as delimiter and sort on the third field.
 	# Finally, select the first entry in that sorted list.
-	OLDEST_INDEX=$({{ ELASTICCURL }} -s -k https://{{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/_cat/indices?h=index\&expand_wildcards=closed | grep -E "(logstash-|so-)" | sort -t- -k3 | head -1)
+	OLDEST_INDEX=$({{ ELASTICCURL }} -s -k https://{{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/_cat/indices?h=index\&expand_wildcards=closed | grep -v "so-case" | grep -E "(logstash-|so-)" | sort -t- -k3 | head -1)
 	
 	# Now that we've determined OLDEST_INDEX, ask Elasticsearch to delete it.
 	{{ ELASTICCURL }} -XDELETE -k https://{{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/${OLDEST_INDEX}

salt/elasticsearch/auth.sls

@@ -4,7 +4,7 @@
   {% set DIGITS = "1234567890" %}
   {% set LOWERCASE = "qwertyuiopasdfghjklzxcvbnm" %}
   {% set UPPERCASE = "QWERTYUIOPASDFGHJKLZXCVBNM" %}
-  {% set SYMBOLS = "~!@#$^&*()-_=+[]|;:,.<>?" %}
+  {% set SYMBOLS = "~!@#^&*()-_=+[]|;:,.<>?" %}
   {% set CHARS = DIGITS~LOWERCASE~UPPERCASE~SYMBOLS %}
   {% set so_elastic_user_pass = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', salt['random.get_str'](72, chars=CHARS)) %}
   {% set so_kibana_user_pass = salt['pillar.get']('elasticsearch:auth:users:so_kibana_user:pass', salt['random.get_str'](72, chars=CHARS)) %}

salt/elasticsearch/base-template.json.jinja

@@ -0,0 +1 @@
+{{ TEMPLATE_CONFIG | tojson(true) }}

salt/elasticsearch/config.map.jinja

@@ -10,7 +10,7 @@
     {% if salt['pillar.get']('nodestab', {}) %}
       {% do ESCONFIG.elasticsearch.config.node.update({'roles': ['master', 'data', 'remote_cluster_client']}) %}
       {% if HIGHLANDER %}
-          {% do ESCONFIG.elasticsearch.config.node.roles.append('ml', 'transform') %}
+          {% do ESCONFIG.elasticsearch.config.node.roles.extend(['ml', 'transform']) %}
       {% endif %}
       {% do ESCONFIG.elasticsearch.config.update({'discovery': {'seed_hosts': [grains.master]}}) %}
       {% for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %}

salt/elasticsearch/files/ingest/rita.beacon

@@ -0,0 +1,127 @@
+{
+  "description": "RITA Beacons",
+  "processors": [
+    {
+      "set": {
+        "field": "_index",
+        "value": "so-rita",
+        "override": true
+      }
+    },
+    {
+      "csv": {
+        "field": "message",
+        "target_fields": [
+          "beacon.score",
+          "source.ip",
+          "destination.ip",
+          "network.connections",
+          "network.average_bytes",
+          "beacon.interval.range",
+          "beacon.size.range",
+          "beacon.interval.top",
+          "beacon.size.top",
+          "beacon.interval.top_count",
+          "beacon.size.top_count",
+          "beacon.interval.skew",
+          "beacon.size.skew",
+          "beacon.interval.dispersion",
+          "beacon.size.dispersion",
+          "network.bytes"
+        ]
+      }
+    },
+    {
+      "convert": {
+        "field": "beacon.score",
+        "type": "float"
+      }
+    },
+    {
+      "convert": {
+        "field": "network.connections",
+        "type": "integer"
+      }
+    },
+    {
+      "convert": {
+        "field": "network.average_bytes",
+        "type": "integer"
+      }
+    },
+    {
+      "convert": {
+        "field": "beacon.interval.range",
+        "type": "integer"
+      }
+    },
+    {
+      "convert": {
+        "field": "beacon.size.range",
+        "type": "integer"
+      }
+    },
+    {
+      "convert": {
+        "field": "beacon.interval.top",
+        "type": "integer"
+      }
+    },
+    {
+      "convert": {
+        "field": "beacon.size.top",
+        "type": "integer"
+      }
+    },
+    {
+      "convert": {
+        "field": "beacon.interval.top_count",
+        "type": "integer"
+      }
+    },
+    {
+      "convert": {
+        "field": "beacon.size.top_count",
+        "type": "integer"
+      }
+    },
+    {
+      "convert": {
+        "field": "beacon.interval.skew",
+        "type": "float"
+      }
+    },
+    {
+      "convert": {
+        "field": "beacon.size.skew",
+        "type": "float"
+      }
+    },
+    {
+      "convert": {
+        "field": "beacon.interval.dispersion",
+        "type": "integer"
+      }
+    },
+    {
+      "convert": {
+        "field": "beacon.size.dispersion",
+        "type": "integer"
+      }
+    },
+    {
+      "convert": {
+        "field": "network.bytes",
+        "type": "integer"
+      }
+    },
+    { "set":         { "if": "ctx.beacon?.score == 1", "field": "dataset",        "value": "alert", "override": true }},
+    { "set":         { "if": "ctx.beacon?.score == 1", "field": "rule.name",        "value": "Potential C2 Beacon Activity", "override": true }},
+    { "set":         { "if": "ctx.beacon?.score == 1", "field": "event.severity",        "value": 3, "override": true }},
+    {
+      "pipeline": {
+        "name": "common"
+      }
+    }
+  ]
+}

salt/elasticsearch/files/ingest/rita.connection

@@ -0,0 +1,36 @@
+{
+  "description": "RITA Connections",
+  "processors": [
+    {
+      "set": {
+        "field": "_index",
+        "value": "so-rita",
+        "override": true
+      }
+    },
+    {
+      "dissect": {
+        "field": "message",
+        "pattern": "%{source.ip},%{destination.ip},%{network.port}:%{network.protocol}:%{network.service},%{connection.duration},%{connection.state}"
+      }
+    },
+    {
+      "convert": {
+        "field": "connection.duration",
+        "type": "float"
+      }
+    },
+    {
+      "set": {
+        "field": "event.duration",
+        "value": "{{ connection.duration }}",
+        "override": true
+      }
+    },
+    {
+      "pipeline": {
+        "name": "common"
+      }
+    }
+  ]
+}

salt/elasticsearch/files/ingest/rita.dns

@@ -0,0 +1,39 @@
+{
+  "description": "RITA DNS",
+  "processors": [
+    {
+      "set": {
+        "field": "_index",
+        "value": "so-rita",
+        "override": true
+      }
+    },
+    {
+      "csv": {
+        "field": "message",
+        "target_fields": [
+          "dns.question.name",
+          "dns.question.subdomain_count",
+          "dns.question.count"
+        ]
+      }
+    },
+    {
+      "convert": {
+        "field": "dns.question.subdomain_count",
+        "type": "integer"
+      }
+    },
+    {
+      "convert": {
+        "field": "dns.question.count",
+        "type": "integer"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "common"
+      }
+    }
+  ]
+}

salt/elasticsearch/files/ingest/suricata.common

@@ -12,7 +12,7 @@
     { "remove":{ "field": "dataset", 						"ignore_failure": true	} },
     { "rename":{ "field": "message2.event_type",		"target_field": "dataset",		"ignore_failure": true 	} },
     { "set":         { "field": "observer.name",        "value": "{{agent.name}}"  } },
-    { "set":         { "field": "ingest.timestamp",        "value": "{{@timestamp}}"  } },
+    { "set":         { "field": "event.ingested",        "value": "{{@timestamp}}"  } },
     { "date": { "field": "message2.timestamp", "target_field": "@timestamp", "formats": ["ISO8601", "UNIX"], "timezone": "UTC", "ignore_failure": true } },
     { "remove":{ "field": "agent", 						"ignore_failure": true	} },
     { "pipeline": { "if": "ctx?.dataset != null", "name": "suricata.{{dataset}}" } }

salt/elasticsearch/files/ingest/syslog

@@ -1,36 +1,157 @@
 {
-  "description" : "syslog",
+  "description" : "syslog pipeline",
   "processors" : [
     {
-        "dissect": {
-                "field": "message",
-                "pattern" : "%{message}",
-                "on_failure": [ { "drop" : { } } ]
-        },
-        "remove": {
-                "field": [ "type", "agent" ],
-                "ignore_failure": true
-        }
+      "dissect": {
+        "field": "message",
+        "pattern" : "%{message}",
+        "on_failure": [ { "drop" : { } } ]
+      },
+      "remove": {
+        "field": [ "type", "agent" ],
+        "ignore_failure": true
+      }
+    }, {
+      "grok": {
+        "field": "message",
+          "patterns": [
+            "^<%{INT:syslog.priority:int}>%{TIMESTAMP_ISO8601:syslog.timestamp} +%{IPORHOST:syslog.host} +%{PROG:syslog.program}(?:\\[%{POSINT:syslog.pid:int}\\])?: %{GREEDYDATA:real_message}$",
+
+            "^<%{INT:syslog.priority}>%{DATA:syslog.timestamp} %{WORD:source.application}(\\[%{DATA:pid}\\])?: %{GREEDYDATA:real_message}$",
+
+            "^%{SYSLOGTIMESTAMP:syslog.timestamp} %{SYSLOGHOST:syslog.host} %{SYSLOGPROG:syslog.program}: CEF:0\\|%{DATA:vendor}\\|%{DATA:product}\\|%{GREEDYDATA:message2}$"
+          ],
+          "ignore_failure": true
+      }
     },
     {
-        "grok":
-                {
-                        "field": "message",
-                        "patterns": [
-                                        "^<%{INT:syslog.priority}>%{DATA:syslog.timestamp} %{WORD:source.application}(\\[%{DATA:pid}\\])?: %{GREEDYDATA:real_message}$",
-                                        "^%{SYSLOGTIMESTAMP:syslog.timestamp} %{SYSLOGHOST:syslog.host} %{SYSLOGPROG:syslog.program}: CEF:0\\|%{DATA:vendor}\\|%{DATA:product}\\|%{GREEDYDATA:message2}$"
-                                    ],
-                        "ignore_failure": true
-                }
+      "convert" : {
+        "if": "ctx?.syslog?.priority != null",
+        "field" : "syslog.priority",
+        "type": "integer"
+      }
     },
-    { "set": { "if": "ctx.source?.application == 'filterlog'", "field": "dataset", "value": "firewall", "ignore_failure": true  } },
-    { "set": { "if": "ctx.vendor != null", "field": "module", "value": "{{ vendor }}", "ignore_failure": true } },
-    { "set": { "if": "ctx.product != null", "field": "dataset", "value": "{{ product }}", "ignore_failure": true } },
-    { "set": { "field": "ingest.timestamp",      "value": "{{ @timestamp }}"       } },
-    { "date": { "if": "ctx.syslog?.timestamp != null", "field": "syslog.timestamp",       "target_field": "@timestamp",   "formats": ["MMM  d HH:mm:ss", "MMM dd HH:mm:ss", "ISO8601", "UNIX"], "ignore_failure": true  } },
-    { "remove":         { "field": ["pid", "program"],  "ignore_missing": true, "ignore_failure": true  } },
-    { "pipeline": { "if": "ctx.vendor != null && ctx.product != null", "name": "{{ vendor }}.{{ product }}", "ignore_failure": true } },
-    { "pipeline": { "if": "ctx.dataset == 'firewall'", "name": "filterlog", "ignore_failure": true } },
-    { "pipeline":       { "name": "common"                                             } }
+    {
+      "script": {
+        "description": "Map syslog priority into facility and level",
+        "lang": "painless",
+        "params" : {
+          "level": [
+            "emerg",
+            "alert",
+            "crit",
+            "err",
+            "warn",
+            "notice",
+            "info",
+            "debug"
+           ],
+           "facility" : [
+             "kern",
+             "user",
+             "mail",
+             "daemon",
+             "auth",
+             "syslog",
+             "lpr",
+             "news",
+             "uucp",
+             "cron",
+             "authpriv",
+             "ftp",
+             "ntp",
+             "security",
+             "console",
+             "solaris-cron",
+             "local0",
+             "local1",
+             "local2",
+             "local3",
+             "local4",
+             "local5",
+             "local6",
+             "local7"
+            ]
+        },
+        "source": "if (ctx['syslog'] != null && ctx['syslog']['priority'] != null) { int p = ctx['syslog']['priority']; int f = p / 8; int l = p - (f * 8); ctx['syslog']['facility_label'] = [ : ]; ctx['syslog']['severity_label'] = [ : ]; ctx['syslog'].put('severity', l); ctx['syslog'].put('severity_label', params.level[l].toUpperCase()); ctx['syslog'].put('facility', f); ctx['syslog'].put('facility_label', params.facility[f].toUpperCase()); }"
+
+      }
+    },
+    {
+      "set": {
+        "if": "ctx.syslog?.host != null",
+        "field": "host.name",
+        "value": "{{ syslog.host }}",
+        "ignore_failure": true
+      }
+    }, {
+      "set": {
+        "if": "ctx.syslog?.program != null",
+        "field": "process.name",
+        "value": "{{ syslog.program }}",
+        "ignore_failure": true
+      }
+    }, {
+      "set": {
+        "if": "ctx.syslog?.pid != null",
+        "field": "process.id",
+        "value": "{{ syslog.pid }}",
+        "ignore_failure": true
+      }
+    }, {
+      "set": {
+        "if": "ctx.source?.application == 'filterlog'",
+        "field": "dataset",
+        "value": "firewall",
+        "ignore_failure": true
+      }
+    }, {
+      "set": {
+        "if": "ctx.vendor != null",
+        "field": "module",
+        "value": "{{ vendor }}",
+        "ignore_failure": true
+      }
+    }, {
+      "set": {
+        "if": "ctx.product != null",
+        "field": "dataset",
+        "value": "{{ product }}",
+        "ignore_failure": true
+      }
+    }, {
+      "set": {
+        "field": "ingest.timestamp",
+        "value": "{{ @timestamp }}"
+      }
+    }, {
+      "date": {
+        "if": "ctx.syslog?.timestamp != null",
+        "field": "syslog.timestamp",
+        "target_field": "@timestamp",
+        "formats": ["MMM  d HH:mm:ss", "MMM dd HH:mm:ss", "ISO8601", "UNIX"],
+        "ignore_failure": true
+      }
+    }, {
+      "remove": {
+        "field": ["pid", "program"],
+        "ignore_missing": true,
+        "ignore_failure": true
+      }
+    }, {
+      "pipeline": {
+        "if": "ctx.vendor != null && ctx.product != null",
+        "name": "{{ vendor }}.{{ product }}",
+        "ignore_failure": true
+      }
+    }, {
+      "pipeline": {
+        "if": "ctx.dataset == 'firewall'",
+        "name": "filterlog",
+        "ignore_failure": true
+      }
+    }, {
+      "pipeline": { "name": "common" }
+    }
   ]
 }

salt/elasticsearch/files/ingest/zeek.common

@@ -1,8 +1,8 @@
 {
   "description" : "zeek.common",
   "processors" : [
-    { "rename":         { "if": "ctx.message2?.ts != null", "field": "@timestamp",      "target_field": "ingest.timestamp",     "ignore_missing": true                                  } },
-    { "set":            { "if": "ctx.message2?.ts == null", "field": "ingest.timestamp",      "value": "{{ @timestamp }}"       } },
+    { "rename":         { "if": "ctx.message2?.ts != null", "field": "@timestamp",      "target_field": "event.ingested",     "ignore_missing": true                                  } },
+    { "set":            { "if": "ctx.message2?.ts == null", "field": "event.ingested",      "value": "{{ @timestamp }}"       } },
     { "rename":         { "field": "message2.uid",              "target_field": "log.id.uid",                  "ignore_missing": true  } },
     { "dot_expander":   { "field": "id.orig_h",                 "path": "message2",                     "ignore_failure": true  } },
     { "dot_expander":   { "field": "id.orig_p",                 "path": "message2",                     "ignore_failure": true  } },

salt/elasticsearch/files/ingest/zeek.dns

@@ -19,10 +19,11 @@
     { "rename": 	{ "field": "message2.RD", 		"target_field": "dns.recursion.desired",			"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.RA", 		"target_field": "dns.recursion.available",			"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.Z",		"target_field": "dns.reserved",			"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.answers", 		"target_field": "dns.answers",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.answers", 		"target_field": "dns.answers.name",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.TTLs", 		"target_field": "dns.ttls",			"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.rejected", 	"target_field": "dns.query.rejected",		"ignore_missing": true 	} },
     { "script": 	{ "lang": "painless", "source": "ctx.dns.query.length = ctx.dns.query.name.length()",	"ignore_failure": true  } },
+    { "set":      { "if": "ctx._index == 'so-zeek'", "field": "_index",      "value": "so-zeek_dns", "override": true   } },
     { "pipeline": { "if": "ctx.dns.query?.name != null && ctx.dns.query.name.contains('.')", "name": "dns.tld" } },
     { "pipeline":       { "name": "zeek.common"											} }
   ]

salt/elasticsearch/files/ingest/zeek.syslog

@@ -4,8 +4,8 @@
     { "remove":         { "field": ["host"],     "ignore_failure": true                                                                  } },
     { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
     { "rename": 	{ "field": "message2.proto", 		"target_field": "network.protocol",		"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.facility", 	"target_field": "syslog.facility",		"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.severity", 	"target_field": "syslog.severity",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.facility", 	"target_field": "syslog.facility_label",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.severity", 	"target_field": "syslog.severity_label",		"ignore_missing": true 	} },
     { "remove":		{ "field": "message",			"ignore_failure": true						} },
     { "rename": 	{ "field": "message2.message", 		"target_field": "message",		"ignore_missing": true 	} },
     { "pipeline":       { "name": "zeek.common"                                                                                   } }

salt/elasticsearch/files/log4j2.properties

@@ -11,10 +11,17 @@ appender.rolling.name = rolling
 appender.rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}.log
 appender.rolling.layout.type = PatternLayout
 appender.rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %.10000m%n
-appender.rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}-%d{yyyy-MM-dd}.log
+appender.rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}-%d{yyyy-MM-dd}.log.gz
 appender.rolling.policies.type = Policies
 appender.rolling.policies.time.type = TimeBasedTriggeringPolicy
 appender.rolling.policies.time.interval = 1
 appender.rolling.policies.time.modulate = true
+appender.rolling.strategy.type = DefaultRolloverStrategy
+appender.rolling.strategy.action.type = Delete
+appender.rolling.strategy.action.basepath = /var/log/elasticsearch
+appender.rolling.strategy.action.condition.type = IfFileName
+appender.rolling.strategy.action.condition.glob = *.gz
+appender.rolling.strategy.action.condition.nested_condition.type = IfLastModified
+appender.rolling.strategy.action.condition.nested_condition.age = 7D
 rootLogger.level = info
 rootLogger.appenderRef.rolling.ref = rolling

salt/elasticsearch/init.sls

@@ -41,7 +41,7 @@ include:
 {% set ROLES = salt['pillar.get']('elasticsearch:roles', {}) %}
 {% from 'elasticsearch/auth.map.jinja' import ELASTICAUTH with context %}
 {% from 'elasticsearch/config.map.jinja' import ESCONFIG with context %}
-
+{% from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS without context %}
 
 vm.max_map_count:
   sysctl.present:
@@ -147,7 +147,7 @@ esingestdir:
 
 estemplatedir:
   file.directory:
-    - name: /opt/so/conf/elasticsearch/templates
+    - name: /opt/so/conf/elasticsearch/templates/index
     - user: 930
     - group: 939
     - makedirs: True
@@ -196,20 +196,48 @@ esyml:
         ESCONFIG: {{ ESCONFIG }}
     - template: jinja
 
-#sync templates to /opt/so/conf/elasticsearch/templates
+escomponenttemplates:
+  file.recurse:
+    - name: /opt/so/conf/elasticsearch/templates/component
+    - source: salt://elasticsearch/templates/component
+    - user: 930
+    - group: 939
+    - onchanges_in:
+      - cmd: so-elasticsearch-templates
+      
+# Auto-generate templates from defaults file
+{% for index, settings in ES_INDEX_SETTINGS.items() %}
+  {% if settings.index_template is defined %}
+es_index_template_{{index}}:
+  file.managed:
+    - name: /opt/so/conf/elasticsearch/templates/index/{{ index }}-template.json
+    - source: salt://elasticsearch/base-template.json.jinja
+    - defaults:
+      TEMPLATE_CONFIG: {{ settings.index_template }}
+    - template: jinja
+    - onchanges_in:
+      - cmd: so-elasticsearch-templates
+  {% endif %}
+{% endfor %}
+
+{% if TEMPLATES %}
+# Sync custom templates to /opt/so/conf/elasticsearch/templates
 {% for TEMPLATE in TEMPLATES %}
 es_template_{{TEMPLATE.split('.')[0] | replace("/","_") }}:
   file.managed:
-    - source: salt://elasticsearch/templates/{{TEMPLATE}}
+    - source: salt://elasticsearch/templates/index/{{TEMPLATE}}
     {% if 'jinja' in TEMPLATE.split('.')[-1] %}
-    - name: /opt/so/conf/elasticsearch/templates/{{TEMPLATE.split('/')[1] | replace(".jinja", "")}}
+    - name: /opt/so/conf/elasticsearch/templates/index/{{TEMPLATE.split('/')[1] | replace(".jinja", "")}}
     - template: jinja
     {% else %}
-    - name: /opt/so/conf/elasticsearch/templates/{{TEMPLATE.split('/')[1]}}
+    - name: /opt/so/conf/elasticsearch/templates/index/{{TEMPLATE.split('/')[1]}}
     {% endif %}
     - user: 930
     - group: 939
+    - onchanges_in:
+      - cmd: so-elasticsearch-templates
 {% endfor %}
+{% endif %}
 
 esroles:
   file.recurse:
@@ -242,6 +270,15 @@ es_repo_dir:
     - require:
       - file: nsmesdir
 
+so-pipelines-reload:
+  file.absent:
+    - name: /opt/so/state/espipelines.txt
+    - onchanges:
+      - file: esingestconf
+      - file: esingestdynamicconf
+      - file: esyml
+      - file: so-elasticsearch-pipelines-script
+
 auth_users:
   file.managed:
     - name: /opt/so/conf/elasticsearch/users.tmp
@@ -332,9 +369,6 @@ so-elasticsearch:
     - watch:
       - file: cacertz
       - file: esyml
-      - file: esingestconf
-      - file: esingestdynamicconf
-      - file: so-elasticsearch-pipelines-script
     - require:
       - file: esyml
       - file: eslog4jfile
@@ -359,19 +393,6 @@ append_so-elasticsearch_so-status.conf:
     - name: /opt/so/conf/so-status/so-status.conf
     - text: so-elasticsearch
 
-so-elasticsearch-pipelines:
-  cmd.run:
-    - name: /usr/sbin/so-elasticsearch-pipelines {{ grains.host }}
-    - onchanges:
-      - file: esingestconf
-      - file: esingestdynamicconf
-      - file: esyml
-      - file: so-elasticsearch-pipelines-script
-    - require:
-      - docker_container: so-elasticsearch
-      - file: so-elasticsearch-pipelines-script
-
-{% if TEMPLATES %}
 so-elasticsearch-templates:
   cmd.run:
     - name: /usr/sbin/so-elasticsearch-templates-load
@@ -380,7 +401,13 @@ so-elasticsearch-templates:
     - require:
       - docker_container: so-elasticsearch
       - file: es_sync_scripts
-{% endif %}
+
+so-elasticsearch-pipelines:
+  cmd.run:
+    - name: /usr/sbin/so-elasticsearch-pipelines {{ grains.host }}
+    - require:
+      - docker_container: so-elasticsearch
+      - file: so-elasticsearch-pipelines-script
 
 so-elasticsearch-roles-load:
   cmd.run:

salt/elasticsearch/template.map.jinja

@@ -0,0 +1,9 @@
+{% import_yaml 'elasticsearch/defaults.yaml' as ESCONFIG with context %}
+{%- set ES_INDEX_SETTINGS = salt['pillar.get']('elasticsearch:index_settings', default=ESCONFIG.elasticsearch.index_settings, merge=True) %}
+{% for index, settings in ES_INDEX_SETTINGS.items() %}
+  {% if settings.index_template is defined %}
+    {% if not settings.get('index_sorting', False) | to_bool and settings.index_template.template.settings.index.sort is defined %}
+      {% do settings.index_template.template.settings.index.pop('sort') %}
+    {% endif %}
+  {% endif %}
+{% endfor %}

salt/elasticsearch/templates/component/ecs/agent.json

@@ -0,0 +1,120 @@
+{
+  "_meta": {
+    "documentation": "https://www.elastic.co/guide/en/ecs/current/ecs-agent.html",
+    "ecs_version": "1.12.2"
+  },
+  "template": {
+    "settings": {
+      "analysis": {
+        "analyzer": {
+          "es_security_analyzer": {
+            "type": "custom",
+            "char_filter": [
+              "whitespace_no_way"
+            ],
+            "filter": [
+              "lowercase",
+              "trim"
+            ],
+            "tokenizer": "keyword"
+          }
+        },
+        "char_filter": {
+          "whitespace_no_way": {
+            "type": "pattern_replace",
+            "pattern": "(\\s)+",
+            "replacement": "$1"
+          }
+        },
+        "filter": {
+          "path_hierarchy_pattern_filter": {
+            "type": "pattern_capture",
+            "preserve_original": true,
+            "patterns": [
+              "((?:[^\\\\]*\\\\)*)(.*)",
+              "((?:[^/]*/)*)(.*)"
+            ]
+          }
+        },
+        "tokenizer": {
+          "path_tokenizer": {
+            "type": "path_hierarchy",
+            "delimiter": "\\"
+          }
+        }
+      }
+    },
+    "mappings": {
+      "properties": {
+        "agent": {
+          "properties": {
+            "build": {
+              "properties": {
+                "original": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                }
+              }
+            },
+            "ephemeral_id": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                }
+              }
+            },
+            "id": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                }
+              }
+            },
+            "name": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                }
+              }
+            },
+            "type": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                }
+              }
+            },
+            "version": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}

salt/elasticsearch/templates/component/ecs/base.json

@@ -0,0 +1,71 @@
+{
+  "_meta": {
+    "documentation": "https://www.elastic.co/guide/en/ecs/current/ecs-base.html",
+    "ecs_version": "1.12.2"
+  },
+  "template": {
+    "settings": {
+      "analysis": {
+        "analyzer": {
+          "es_security_analyzer": {
+            "type": "custom",
+            "char_filter": [
+              "whitespace_no_way"
+            ],
+            "filter": [
+              "lowercase",
+              "trim"
+            ],
+            "tokenizer": "keyword"
+          }
+        },
+        "char_filter": {
+          "whitespace_no_way": {
+            "type": "pattern_replace",
+            "pattern": "(\\s)+",
+            "replacement": "$1"
+          }
+        },
+        "filter": {
+          "path_hierarchy_pattern_filter": {
+            "type": "pattern_capture",
+            "preserve_original": true,
+            "patterns": [
+              "((?:[^\\\\]*\\\\)*)(.*)",
+              "((?:[^/]*/)*)(.*)"
+            ]
+          }
+        },
+        "tokenizer": {
+          "path_tokenizer": {
+            "type": "path_hierarchy",
+            "delimiter": "\\"
+          }
+        }
+      }
+    },
+    "mappings": {
+      "properties": {
+        "@timestamp": {
+          "type": "date"
+        },
+        "labels": {
+          "type": "object"
+        },
+        "message": {
+          "type": "match_only_text"
+        },
+        "tags": {
+          "ignore_above": 1024,
+          "type": "keyword",
+          "fields": {
+            "security": {
+              "type": "text",
+              "analyzer": "es_security_analyzer"
+            }
+          }
+        }
+      }
+    }
+  }
+}

salt/elasticsearch/templates/component/ecs/client.json

@@ -0,0 +1,374 @@
+{
+  "_meta": {
+    "documentation": "https://www.elastic.co/guide/en/ecs/current/ecs-client.html",
+    "ecs_version": "1.12.2"
+  },
+  "template": {
+    "settings": {
+      "analysis": {
+        "analyzer": {
+          "es_security_analyzer": {
+            "type": "custom",
+            "char_filter": [
+              "whitespace_no_way"
+            ],
+            "filter": [
+              "lowercase",
+              "trim"
+            ],
+            "tokenizer": "keyword"
+          }
+        },
+        "char_filter": {
+          "whitespace_no_way": {
+            "type": "pattern_replace",
+            "pattern": "(\\s)+",
+            "replacement": "$1"
+          }
+        },
+        "filter": {
+          "path_hierarchy_pattern_filter": {
+            "type": "pattern_capture",
+            "preserve_original": true,
+            "patterns": [
+              "((?:[^\\\\]*\\\\)*)(.*)",
+              "((?:[^/]*/)*)(.*)"
+            ]
+          }
+        },
+        "tokenizer": {
+          "path_tokenizer": {
+            "type": "path_hierarchy",
+            "delimiter": "\\"
+          }
+        }
+      }
+    },
+    "mappings": {
+      "properties": {
+        "client": {
+          "properties": {
+            "address": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                }
+              }
+            },
+            "as": {
+              "properties": {
+                "number": {
+                  "type": "long"
+                },
+                "organization": {
+                  "properties": {
+                    "name": {
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      },
+                      "ignore_above": 1024,
+                      "type": "keyword"
+                    }
+                  }
+                }
+              }
+            },
+            "bytes": {
+              "type": "long"
+            },
+            "domain": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                }
+              }
+            },
+            "geo": {
+              "properties": {
+                "city_name": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "continent_code": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "continent_name": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "country_iso_code": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "country_name": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "location": {
+                  "type": "geo_point"
+                },
+                "name": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "postal_code": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "region_iso_code": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "region_name": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "timezone": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                }
+              }
+            },
+            "ip": {
+              "type": "ip"
+            },
+            "mac": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                }
+              }
+            },
+            "nat": {
+              "properties": {
+                "ip": {
+                  "type": "ip"
+                },
+                "port": {
+                  "type": "long"
+                }
+              }
+            },
+            "packets": {
+              "type": "long"
+            },
+            "port": {
+              "type": "long"
+            },
+            "registered_domain": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                }
+              }
+            },
+            "subdomain": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                }
+              }
+            },
+            "top_level_domain": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                }
+              }
+            },
+            "user": {
+              "properties": {
+                "domain": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "email": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "full_name": {
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  },
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "group": {
+                  "properties": {
+                    "domain": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "id": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "name": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    }
+                  }
+                },
+                "hash": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "id": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "name": {
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  },
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "roles": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}

salt/elasticsearch/templates/component/ecs/cloud.json

@@ -0,0 +1,186 @@
+{
+  "_meta": {
+    "documentation": "https://www.elastic.co/guide/en/ecs/current/ecs-cloud.html",
+    "ecs_version": "1.12.2"
+  },
+  "template": {
+    "settings": {
+      "analysis": {
+        "analyzer": {
+          "es_security_analyzer": {
+            "type": "custom",
+            "char_filter": [
+              "whitespace_no_way"
+            ],
+            "filter": [
+              "lowercase",
+              "trim"
+            ],
+            "tokenizer": "keyword"
+          }
+        },
+        "char_filter": {
+          "whitespace_no_way": {
+            "type": "pattern_replace",
+            "pattern": "(\\s)+",
+            "replacement": "$1"
+          }
+        },
+        "filter": {
+          "path_hierarchy_pattern_filter": {
+            "type": "pattern_capture",
+            "preserve_original": true,
+            "patterns": [
+              "((?:[^\\\\]*\\\\)*)(.*)",
+              "((?:[^/]*/)*)(.*)"
+            ]
+          }
+        },
+        "tokenizer": {
+          "path_tokenizer": {
+            "type": "path_hierarchy",
+            "delimiter": "\\"
+          }
+        }
+      }
+    },
+    "mappings": {
+      "properties": {
+        "cloud": {
+          "properties": {
+            "account": {
+              "properties": {
+                "id": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "name": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                }
+              }
+            },
+            "availability_zone": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                }
+              }
+            },
+            "instance": {
+              "properties": {
+                "id": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "name": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                }
+              }
+            },
+            "machine": {
+              "properties": {
+                "type": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                }
+              }
+            },
+            "project": {
+              "properties": {
+                "id": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "name": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                }
+              }
+            },
+            "provider": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                }
+              }
+            },
+            "region": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                }
+              }
+            },
+            "service": {
+              "properties": {
+                "name": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}

salt/elasticsearch/templates/component/ecs/container.json

@@ -0,0 +1,113 @@
+{
+  "_meta": {
+    "documentation": "https://www.elastic.co/guide/en/ecs/current/ecs-container.html",
+    "ecs_version": "1.12.2"
+  },
+  "template": {
+    "settings": {
+      "analysis": {
+        "analyzer": {
+          "es_security_analyzer": {
+            "type": "custom",
+            "char_filter": [
+              "whitespace_no_way"
+            ],
+            "filter": [
+              "lowercase",
+              "trim"
+            ],
+            "tokenizer": "keyword"
+          }
+        },
+        "char_filter": {
+          "whitespace_no_way": {
+            "type": "pattern_replace",
+            "pattern": "(\\s)+",
+            "replacement": "$1"
+          }
+        },
+        "filter": {
+          "path_hierarchy_pattern_filter": {
+            "type": "pattern_capture",
+            "preserve_original": true,
+            "patterns": [
+              "((?:[^\\\\]*\\\\)*)(.*)",
+              "((?:[^/]*/)*)(.*)"
+            ]
+          }
+        },
+        "tokenizer": {
+          "path_tokenizer": {
+            "type": "path_hierarchy",
+            "delimiter": "\\"
+          }
+        }
+      }
+    },
+    "mappings": {
+      "properties": {
+        "container": {
+          "properties": {
+            "id": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                }
+              }
+            },
+            "image": {
+              "properties": {
+                "name": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "tag": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                }
+              }
+            },
+            "labels": {
+              "type": "object"
+            },
+            "name": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                }
+              }
+            },
+            "runtime": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}

salt/elasticsearch/templates/component/ecs/cyberark.json

@@ -0,0 +1,723 @@
+{
+  "_meta": {
+    "documentation": "https://www.elastic.co/guide/en/ecs/current/ecs-base.html",
+    "ecs_version": "1.12.2"
+  },
+  "template": {
+    "settings": {
+      "analysis": {
+        "analyzer": {
+          "es_security_analyzer": {
+            "type": "custom",
+            "char_filter": [
+              "whitespace_no_way"
+            ],
+            "filter": [
+              "lowercase",
+              "trim"
+            ],
+            "tokenizer": "keyword"
+          }
+        },
+        "char_filter": {
+          "whitespace_no_way": {
+            "type": "pattern_replace",
+            "pattern": "(\\s)+",
+            "replacement": "$1"
+          }
+        },
+        "filter": {
+          "path_hierarchy_pattern_filter": {
+            "type": "pattern_capture",
+            "preserve_original": true,
+            "patterns": [
+              "((?:[^\\\\]*\\\\)*)(.*)",
+              "((?:[^/]*/)*)(.*)"
+            ]
+          }
+        },
+        "tokenizer": {
+          "path_tokenizer": {
+            "type": "path_hierarchy",
+            "delimiter": "\\"
+          }
+        }
+      }
+    },
+    "mappings": {
+      "properties": {
+        "cyberarkpas": {
+          "properties": {
+            "audit": {
+              "properties": {
+                "action": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "ca_properties": {
+                  "properties": {
+                    "address": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "cpm_disabled": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "cpm_error_details": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "cpm_status": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "creation_method": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "customer": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "database": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "device_type": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "dual_account_status": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "group_name": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "in_process": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "index": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "last_fail_date": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "last_success_change": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "last_success_reconciliation": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "last_success_verification": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "last_task": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "logon_domain": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "other": {
+                      "type": "flattened"
+                    },
+                    "policy_id": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "port": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "privcloud": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "reset_immediately": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "retries_count": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "sequence_id": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "tags": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "user_dn": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "user_name": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "virtual_username": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    }
+                  }
+                },
+                "category": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "desc": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "extra_details": {
+                  "properties": {
+                    "ad_process_id": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "ad_process_name": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "application_type": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "command": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "connection_component_id": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "dst_host": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "logon_account": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "managed_account": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "other": {
+                      "type": "flattened"
+                    },
+                    "process_id": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "process_name": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "protocol": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "psmid": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "session_duration": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "session_id": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "src_host": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "username": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    }
+                  }
+                },
+                "file": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "gateway_station": {
+                  "type": "ip"
+                },
+                "hostname": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "iso_timestamp": {
+                  "type": "date"
+                },
+                "issuer": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "location": {
+                  "doc_values": false,
+                  "ignore_above": 4096,
+                  "index": false,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "message": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "message_id": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "product": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "pvwa_details": {
+                  "type": "flattened"
+                },
+                "raw": {
+                  "doc_values": false,
+                  "ignore_above": 4096,
+                  "index": false,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "reason": {
+                  "norms": false,
+                  "type": "text",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "rfc5424": {
+                  "type": "boolean"
+                },
+                "safe": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "severity": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "source_user": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "station": {
+                  "type": "ip"
+                },
+                "target_user": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "timestamp": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "vendor": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "version": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}

salt/elasticsearch/templates/component/ecs/data_stream.json

@@ -0,0 +1,65 @@
+{
+  "_meta": {
+    "documentation": "https://www.elastic.co/guide/en/ecs/current/ecs-data_stream.html",
+    "ecs_version": "1.12.2"
+  },
+  "template": {
+    "settings": {
+      "analysis": {
+        "analyzer": {
+          "es_security_analyzer": {
+            "type": "custom",
+            "char_filter": [
+              "whitespace_no_way"
+            ],
+            "filter": [
+              "lowercase",
+              "trim"
+            ],
+            "tokenizer": "keyword"
+          }
+        },
+        "char_filter": {
+          "whitespace_no_way": {
+            "type": "pattern_replace",
+            "pattern": "(\\s)+",
+            "replacement": "$1"
+          }
+        },
+        "filter": {
+          "path_hierarchy_pattern_filter": {
+            "type": "pattern_capture",
+            "preserve_original": true,
+            "patterns": [
+              "((?:[^\\\\]*\\\\)*)(.*)",
+              "((?:[^/]*/)*)(.*)"
+            ]
+          }
+        },
+        "tokenizer": {
+          "path_tokenizer": {
+            "type": "path_hierarchy",
+            "delimiter": "\\"
+          }
+        }
+      }
+    },
+    "mappings": {
+      "properties": {
+        "data_stream": {
+          "properties": {
+            "dataset": {
+              "type": "constant_keyword"
+            },
+            "namespace": {
+              "type": "constant_keyword"
+            },
+            "type": {
+              "type": "constant_keyword"
+            }
+          }
+        }
+      }
+    }
+  }
+}

salt/elasticsearch/templates/component/ecs/destination.json

@@ -0,0 +1,374 @@
+{
+  "_meta": {
+    "documentation": "https://www.elastic.co/guide/en/ecs/current/ecs-destination.html",
+    "ecs_version": "1.12.2"
+  },
+  "template": {
+    "settings": {
+      "analysis": {
+        "analyzer": {
+          "es_security_analyzer": {
+            "type": "custom",
+            "char_filter": [
+              "whitespace_no_way"
+            ],
+            "filter": [
+              "lowercase",
+              "trim"
+            ],
+            "tokenizer": "keyword"
+          }
+        },
+        "char_filter": {
+          "whitespace_no_way": {
+            "type": "pattern_replace",
+            "pattern": "(\\s)+",
+            "replacement": "$1"
+          }
+        },
+        "filter": {
+          "path_hierarchy_pattern_filter": {
+            "type": "pattern_capture",
+            "preserve_original": true,
+            "patterns": [
+              "((?:[^\\\\]*\\\\)*)(.*)",
+              "((?:[^/]*/)*)(.*)"
+            ]
+          }
+        },
+        "tokenizer": {
+          "path_tokenizer": {
+            "type": "path_hierarchy",
+            "delimiter": "\\"
+          }
+        }
+      }
+    },
+    "mappings": {
+      "properties": {
+        "destination": {
+          "properties": {
+            "address": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                }
+              }
+            },
+            "as": {
+              "properties": {
+                "number": {
+                  "type": "long"
+                },
+                "organization": {
+                  "properties": {
+                    "name": {
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      },
+                      "ignore_above": 1024,
+                      "type": "keyword"
+                    }
+                  }
+                }
+              }
+            },
+            "bytes": {
+              "type": "long"
+            },
+            "domain": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                }
+              }
+            },
+            "geo": {
+              "properties": {
+                "city_name": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "continent_code": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "continent_name": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "country_iso_code": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "country_name": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "location": {
+                  "type": "geo_point"
+                },
+                "name": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "postal_code": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "region_iso_code": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "region_name": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "timezone": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                }
+              }
+            },
+            "ip": {
+              "type": "ip"
+            },
+            "mac": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                }
+              }
+            },
+            "nat": {
+              "properties": {
+                "ip": {
+                  "type": "ip"
+                },
+                "port": {
+                  "type": "long"
+                }
+              }
+            },
+            "packets": {
+              "type": "long"
+            },
+            "port": {
+              "type": "long"
+            },
+            "registered_domain": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                }
+              }
+            },
+            "subdomain": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                }
+              }
+            },
+            "top_level_domain": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                }
+              }
+            },
+            "user": {
+              "properties": {
+                "domain": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "email": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "full_name": {
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  },
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "group": {
+                  "properties": {
+                    "domain": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "id": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "name": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    }
+                  }
+                },
+                "hash": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "id": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "name": {
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  },
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "roles": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}

salt/elasticsearch/templates/component/ecs/dll.json

@@ -0,0 +1,270 @@
+{
+  "_meta": {
+    "documentation": "https://www.elastic.co/guide/en/ecs/current/ecs-dll.html",
+    "ecs_version": "1.12.2"
+  },
+  "template": {
+    "settings": {
+      "analysis": {
+        "analyzer": {
+          "es_security_analyzer": {
+            "type": "custom",
+            "char_filter": [
+              "whitespace_no_way"
+            ],
+            "filter": [
+              "lowercase",
+              "trim"
+            ],
+            "tokenizer": "keyword"
+          }
+        },
+        "char_filter": {
+          "whitespace_no_way": {
+            "type": "pattern_replace",
+            "pattern": "(\\s)+",
+            "replacement": "$1"
+          }
+        },
+        "filter": {
+          "path_hierarchy_pattern_filter": {
+            "type": "pattern_capture",
+            "preserve_original": true,
+            "patterns": [
+              "((?:[^\\\\]*\\\\)*)(.*)",
+              "((?:[^/]*/)*)(.*)"
+            ]
+          }
+        },
+        "tokenizer": {
+          "path_tokenizer": {
+            "type": "path_hierarchy",
+            "delimiter": "\\"
+          }
+        }
+      }
+    },
+    "mappings": {
+      "properties": {
+        "dll": {
+          "properties": {
+            "code_signature": {
+              "properties": {
+                "digest_algorithm": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "exists": {
+                  "type": "boolean"
+                },
+                "signing_id": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "status": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "subject_name": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "team_id": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "timestamp": {
+                  "type": "date"
+                },
+                "trusted": {
+                  "type": "boolean"
+                },
+                "valid": {
+                  "type": "boolean"
+                }
+              }
+            },
+            "hash": {
+              "properties": {
+                "md5": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "sha1": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "sha256": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "sha512": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "ssdeep": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                }
+              }
+            },
+            "name": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                }
+              }
+            },
+            "path": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                }
+              }
+            },
+            "pe": {
+              "properties": {
+                "architecture": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "company": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "description": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "file_version": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "imphash": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "original_file_name": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "product": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}

salt/elasticsearch/templates/component/ecs/dns.json

@@ -0,0 +1,221 @@
+{
+  "_meta": {
+    "documentation": "https://www.elastic.co/guide/en/ecs/current/ecs-dns.html",
+    "ecs_version": "1.12.2"
+  },
+  "template": {
+    "settings": {
+      "analysis": {
+        "analyzer": {
+          "es_security_analyzer": {
+            "type": "custom",
+            "char_filter": [
+              "whitespace_no_way"
+            ],
+            "filter": [
+              "lowercase",
+              "trim"
+            ],
+            "tokenizer": "keyword"
+          }
+        },
+        "char_filter": {
+          "whitespace_no_way": {
+            "type": "pattern_replace",
+            "pattern": "(\\s)+",
+            "replacement": "$1"
+          }
+        },
+        "filter": {
+          "path_hierarchy_pattern_filter": {
+            "type": "pattern_capture",
+            "preserve_original": true,
+            "patterns": [
+              "((?:[^\\\\]*\\\\)*)(.*)",
+              "((?:[^/]*/)*)(.*)"
+            ]
+          }
+        },
+        "tokenizer": {
+          "path_tokenizer": {
+            "type": "path_hierarchy",
+            "delimiter": "\\"
+          }
+        }
+      }
+    },
+    "mappings": {
+      "properties": {
+        "dns": {
+          "properties": {
+            "answers": {
+              "properties": {
+                "class": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "data": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "name": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "ttl": {
+                  "type": "long"
+                },
+                "type": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                }
+              },
+              "type": "object"
+            },
+            "header_flags": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                }
+              }
+            },
+            "id": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                }
+              }
+            },
+            "op_code": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                }
+              }
+            },
+            "question": {
+              "properties": {
+                "class": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "name": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "registered_domain": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "subdomain": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "top_level_domain": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "type": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                }
+              }
+            },
+            "resolved_ip": {
+              "type": "ip"
+            },
+            "response_code": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                }
+              }
+            },
+            "type": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}

salt/elasticsearch/templates/component/ecs/ecs.json

@@ -0,0 +1,66 @@
+{
+  "_meta": {
+    "documentation": "https://www.elastic.co/guide/en/ecs/current/ecs-ecs.html",
+    "ecs_version": "1.12.2"
+  },
+  "template": {
+    "settings": {
+      "analysis": {
+        "analyzer": {
+          "es_security_analyzer": {
+            "type": "custom",
+            "char_filter": [
+              "whitespace_no_way"
+            ],
+            "filter": [
+              "lowercase",
+              "trim"
+            ],
+            "tokenizer": "keyword"
+          }
+        },
+        "char_filter": {
+          "whitespace_no_way": {
+            "type": "pattern_replace",
+            "pattern": "(\\s)+",
+            "replacement": "$1"
+          }
+        },
+        "filter": {
+          "path_hierarchy_pattern_filter": {
+            "type": "pattern_capture",
+            "preserve_original": true,
+            "patterns": [
+              "((?:[^\\\\]*\\\\)*)(.*)",
+              "((?:[^/]*/)*)(.*)"
+            ]
+          }
+        },
+        "tokenizer": {
+          "path_tokenizer": {
+            "type": "path_hierarchy",
+            "delimiter": "\\"
+          }
+        }
+      }
+    },
+    "mappings": {
+      "properties": {
+        "ecs": {
+          "properties": {
+            "version": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}

salt/elasticsearch/templates/component/ecs/elasticsearch.json

@@ -0,0 +1,71 @@
+{
+  "_meta": {
+    "documentation": "https://www.elastic.co/guide/en/ecs/current/ecs-base.html",
+    "ecs_version": "1.12.2"
+  },
+  "template": {
+    "settings": {
+      "analysis": {
+        "analyzer": {
+          "es_security_analyzer": {
+            "type": "custom",
+            "char_filter": [
+              "whitespace_no_way"
+            ],
+            "filter": [
+              "lowercase",
+              "trim"
+            ],
+            "tokenizer": "keyword"
+          }
+        },
+        "char_filter": {
+          "whitespace_no_way": {
+            "type": "pattern_replace",
+            "pattern": "(\\s)+",
+            "replacement": "$1"
+          }
+        },
+        "filter": {
+          "path_hierarchy_pattern_filter": {
+            "type": "pattern_capture",
+            "preserve_original": true,
+            "patterns": [
+              "((?:[^\\\\]*\\\\)*)(.*)",
+              "((?:[^/]*/)*)(.*)"
+            ]
+          }
+        },
+        "tokenizer": {
+          "path_tokenizer": {
+            "type": "path_hierarchy",
+            "delimiter": "\\"
+          }
+        }
+      }
+    },
+    "mappings": {
+      "properties": {
+        "@timestamp": {
+          "type": "date"
+        },
+        "labels": {
+          "type": "object"
+        },
+        "message": {
+          "type": "match_only_text"
+        },
+        "tags": {
+          "ignore_above": 1024,
+          "type": "keyword",
+          "fields": {
+            "security": {
+              "type": "text",
+              "analyzer": "es_security_analyzer"
+            }
+          }
+        }
+      }
+    }
+  }
+}

salt/elasticsearch/templates/component/ecs/error.json

@@ -0,0 +1,101 @@
+{
+  "_meta": {
+    "documentation": "https://www.elastic.co/guide/en/ecs/current/ecs-error.html",
+    "ecs_version": "1.12.2"
+  },
+  "template": {
+    "settings": {
+      "analysis": {
+        "analyzer": {
+          "es_security_analyzer": {
+            "type": "custom",
+            "char_filter": [
+              "whitespace_no_way"
+            ],
+            "filter": [
+              "lowercase",
+              "trim"
+            ],
+            "tokenizer": "keyword"
+          }
+        },
+        "char_filter": {
+          "whitespace_no_way": {
+            "type": "pattern_replace",
+            "pattern": "(\\s)+",
+            "replacement": "$1"
+          }
+        },
+        "filter": {
+          "path_hierarchy_pattern_filter": {
+            "type": "pattern_capture",
+            "preserve_original": true,
+            "patterns": [
+              "((?:[^\\\\]*\\\\)*)(.*)",
+              "((?:[^/]*/)*)(.*)"
+            ]
+          }
+        },
+        "tokenizer": {
+          "path_tokenizer": {
+            "type": "path_hierarchy",
+            "delimiter": "\\"
+          }
+        }
+      }
+    },
+    "mappings": {
+      "properties": {
+        "error": {
+          "properties": {
+            "code": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                }
+              }
+            },
+            "id": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                }
+              }
+            },
+            "message": {
+              "type": "match_only_text"
+            },
+            "stack_trace": {
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                },
+                "text": {
+                  "type": "match_only_text"
+                }
+              },
+              "type": "wildcard"
+            },
+            "type": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}

salt/elasticsearch/templates/component/ecs/event.json

@@ -0,0 +1,254 @@
+{
+  "_meta": {
+    "documentation": "https://www.elastic.co/guide/en/ecs/current/ecs-event.html",
+    "ecs_version": "1.12.2"
+  },
+  "template": {
+    "settings": {
+      "analysis": {
+        "analyzer": {
+          "es_security_analyzer": {
+            "type": "custom",
+            "char_filter": [
+              "whitespace_no_way"
+            ],
+            "filter": [
+              "lowercase",
+              "trim"
+            ],
+            "tokenizer": "keyword"
+          }
+        },
+        "char_filter": {
+          "whitespace_no_way": {
+            "type": "pattern_replace",
+            "pattern": "(\\s)+",
+            "replacement": "$1"
+          }
+        },
+        "filter": {
+          "path_hierarchy_pattern_filter": {
+            "type": "pattern_capture",
+            "preserve_original": true,
+            "patterns": [
+              "((?:[^\\\\]*\\\\)*)(.*)",
+              "((?:[^/]*/)*)(.*)"
+            ]
+          }
+        },
+        "tokenizer": {
+          "path_tokenizer": {
+            "type": "path_hierarchy",
+            "delimiter": "\\"
+          }
+        }
+      }
+    },
+    "mappings": {
+      "properties": {
+        "event": {
+          "properties": {
+            "action": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                }
+              }
+            },
+            "agent_id_status": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                }
+              }
+            },
+            "category": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                }
+              }
+            },
+            "code": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                }
+              }
+            },
+            "created": {
+              "type": "date"
+            },
+            "dataset": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                }
+              }
+            },
+            "duration": {
+              "type": "long"
+            },
+            "end": {
+              "type": "date"
+            },
+            "hash": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                }
+              }
+            },
+            "id": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                }
+              }
+            },
+            "ingested": {
+              "type": "date"
+            },
+            "kind": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                }
+              }
+            },
+            "module": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                }
+              }
+            },
+            "original": {
+              "doc_values": false,
+              "index": false,
+              "type": "keyword",
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                }
+              }
+            },
+            "outcome": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                }
+              }
+            },
+            "provider": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                }
+              }
+            },
+            "reason": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                }
+              }
+            },
+            "reference": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                }
+              }
+            },
+            "risk_score": {
+              "type": "float"
+            },
+            "risk_score_norm": {
+              "type": "float"
+            },
+            "sequence": {
+              "type": "long"
+            },
+            "severity": {
+              "type": "long"
+            },
+            "start": {
+              "type": "date"
+            },
+            "timezone": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                }
+              }
+            },
+            "type": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                }
+              }
+            },
+            "url": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}

salt/elasticsearch/templates/component/ecs/file.json

@@ -0,0 +1,886 @@
+{
+  "_meta": {
+    "documentation": "https://www.elastic.co/guide/en/ecs/current/ecs-file.html",
+    "ecs_version": "1.12.2"
+  },
+  "template": {
+    "settings": {
+      "analysis": {
+        "analyzer": {
+          "es_security_analyzer": {
+            "type": "custom",
+            "char_filter": [
+              "whitespace_no_way"
+            ],
+            "filter": [
+              "lowercase",
+              "trim"
+            ],
+            "tokenizer": "keyword"
+          }
+        },
+        "char_filter": {
+          "whitespace_no_way": {
+            "type": "pattern_replace",
+            "pattern": "(\\s)+",
+            "replacement": "$1"
+          }
+        },
+        "filter": {
+          "path_hierarchy_pattern_filter": {
+            "type": "pattern_capture",
+            "preserve_original": true,
+            "patterns": [
+              "((?:[^\\\\]*\\\\)*)(.*)",
+              "((?:[^/]*/)*)(.*)"
+            ]
+          }
+        },
+        "tokenizer": {
+          "path_tokenizer": {
+            "type": "path_hierarchy",
+            "delimiter": "\\"
+          }
+        }
+      }
+    },
+    "mappings": {
+      "properties": {
+        "file": {
+          "properties": {
+            "accessed": {
+              "type": "date"
+            },
+            "attributes": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                }
+              }
+            },
+            "code_signature": {
+              "properties": {
+                "digest_algorithm": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "exists": {
+                  "type": "boolean"
+                },
+                "signing_id": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "status": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "subject_name": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "team_id": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "timestamp": {
+                  "type": "date"
+                },
+                "trusted": {
+                  "type": "boolean"
+                },
+                "valid": {
+                  "type": "boolean"
+                }
+              }
+            },
+            "created": {
+              "type": "date"
+            },
+            "ctime": {
+              "type": "date"
+            },
+            "device": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                }
+              }
+            },
+            "directory": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                }
+              }
+            },
+            "drive_letter": {
+              "ignore_above": 1,
+              "type": "keyword",
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                }
+              }
+            },
+            "elf": {
+              "properties": {
+                "architecture": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "byte_order": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "cpu_type": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "creation_date": {
+                  "type": "date"
+                },
+                "exports": {
+                  "type": "flattened"
+                },
+                "header": {
+                  "properties": {
+                    "abi_version": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "class": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "data": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "entrypoint": {
+                      "type": "long"
+                    },
+                    "object_version": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "os_abi": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "type": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "version": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    }
+                  }
+                },
+                "imports": {
+                  "type": "flattened"
+                },
+                "sections": {
+                  "properties": {
+                    "chi2": {
+                      "type": "long"
+                    },
+                    "entropy": {
+                      "type": "long"
+                    },
+                    "flags": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "name": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "physical_offset": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "physical_size": {
+                      "type": "long"
+                    },
+                    "type": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "virtual_address": {
+                      "type": "long"
+                    },
+                    "virtual_size": {
+                      "type": "long"
+                    }
+                  },
+                  "type": "nested"
+                },
+                "segments": {
+                  "properties": {
+                    "sections": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "type": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    }
+                  },
+                  "type": "nested"
+                },
+                "shared_libraries": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "telfhash": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                }
+              }
+            },
+            "extension": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                }
+              }
+            },
+            "fork_name": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                }
+              }
+            },
+            "gid": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                }
+              }
+            },
+            "group": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                }
+              }
+            },
+            "hash": {
+              "properties": {
+                "md5": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "sha1": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "sha256": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "sha512": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "ssdeep": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                }
+              }
+            },
+            "inode": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                }
+              }
+            },
+            "mime_type": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                }
+              }
+            },
+            "mode": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                }
+              }
+            },
+            "mtime": {
+              "type": "date"
+            },
+            "name": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                }
+              }
+            },
+            "owner": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                }
+              }
+            },
+            "path": {
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                }
+              },
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "pe": {
+              "properties": {
+                "architecture": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "company": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "description": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "file_version": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "imphash": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "original_file_name": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "product": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                }
+              }
+            },
+            "size": {
+              "type": "long"
+            },
+            "target_path": {
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                }
+              },
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "type": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                }
+              }
+            },
+            "uid": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "security": {
+                  "type": "text",
+                  "analyzer": "es_security_analyzer"
+                }
+              }
+            },
+            "x509": {
+              "properties": {
+                "alternative_names": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "issuer": {
+                  "properties": {
+                    "common_name": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "country": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "distinguished_name": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "locality": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "organization": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "organizational_unit": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "state_or_province": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    }
+                  }
+                },
+                "not_after": {
+                  "type": "date"
+                },
+                "not_before": {
+                  "type": "date"
+                },
+                "public_key_algorithm": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "public_key_curve": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "public_key_exponent": {
+                  "doc_values": false,
+                  "index": false,
+                  "type": "long"
+                },
+                "public_key_size": {
+                  "type": "long"
+                },
+                "serial_number": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "signature_algorithm": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "subject": {
+                  "properties": {
+                    "common_name": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "country": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "distinguished_name": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "locality": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "organization": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "organizational_unit": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "state_or_province": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    }
+                  }
+                },
+                "version_number": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}

salt/elasticsearch/templates/component/ecs/gcp.json

@@ -0,0 +1,553 @@
+{
+  "_meta": {
+    "documentation": "https://www.elastic.co/guide/en/ecs/current/ecs-base.html",
+    "ecs_version": "1.12.2"
+  },
+  "template": {
+    "settings": {
+      "analysis": {
+        "analyzer": {
+          "es_security_analyzer": {
+            "type": "custom",
+            "char_filter": [
+              "whitespace_no_way"
+            ],
+            "filter": [
+              "lowercase",
+              "trim"
+            ],
+            "tokenizer": "keyword"
+          }
+        },
+        "char_filter": {
+          "whitespace_no_way": {
+            "type": "pattern_replace",
+            "pattern": "(\\s)+",
+            "replacement": "$1"
+          }
+        },
+        "filter": {
+          "path_hierarchy_pattern_filter": {
+            "type": "pattern_capture",
+            "preserve_original": true,
+            "patterns": [
+              "((?:[^\\\\]*\\\\)*)(.*)",
+              "((?:[^/]*/)*)(.*)"
+            ]
+          }
+        },
+        "tokenizer": {
+          "path_tokenizer": {
+            "type": "path_hierarchy",
+            "delimiter": "\\"
+          }
+        }
+      }
+    },
+    "mappings": {
+      "properties": {
+        "gcp": {
+          "properties": {
+            "audit": {
+              "properties": {
+                "authentication_info": {
+                  "properties": {
+                    "authority_selector": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "principal_email": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    }
+                  }
+                },
+                "method_name": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "num_response_items": {
+                  "type": "long"
+                },
+                "request": {
+                  "properties": {
+                    "filter": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "name": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "proto_name": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "resource_name": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    }
+                  }
+                },
+                "request_metadata": {
+                  "properties": {
+                    "caller_ip": {
+                      "type": "ip"
+                    },
+                    "caller_supplied_user_agent": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    }
+                  }
+                },
+                "resource_location": {
+                  "properties": {
+                    "current_locations": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    }
+                  }
+                },
+                "resource_name": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "response": {
+                  "properties": {
+                    "details": {
+                      "properties": {
+                        "group": {
+                          "ignore_above": 1024,
+                          "type": "keyword",
+                          "fields": {
+                            "security": {
+                              "type": "text",
+                              "analyzer": "es_security_analyzer"
+                            }
+                          }
+                        },
+                        "kind": {
+                          "ignore_above": 1024,
+                          "type": "keyword",
+                          "fields": {
+                            "security": {
+                              "type": "text",
+                              "analyzer": "es_security_analyzer"
+                            }
+                          }
+                        },
+                        "name": {
+                          "ignore_above": 1024,
+                          "type": "keyword",
+                          "fields": {
+                            "security": {
+                              "type": "text",
+                              "analyzer": "es_security_analyzer"
+                            }
+                          }
+                        },
+                        "uid": {
+                          "ignore_above": 1024,
+                          "type": "keyword",
+                          "fields": {
+                            "security": {
+                              "type": "text",
+                              "analyzer": "es_security_analyzer"
+                            }
+                          }
+                        }
+                      }
+                    },
+                    "proto_name": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "status": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    }
+                  }
+                },
+                "service_name": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "status": {
+                  "properties": {
+                    "code": {
+                      "type": "long"
+                    },
+                    "message": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    }
+                  }
+                },
+                "type": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                }
+              }
+            },
+            "destination": {
+              "properties": {
+                "instance": {
+                  "properties": {
+                    "project_id": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "region": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "zone": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    }
+                  }
+                },
+                "vpc": {
+                  "properties": {
+                    "project_id": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "subnetwork_name": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "vpc_name": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    }
+                  }
+                }
+              }
+            },
+            "firewall": {
+              "properties": {
+                "rule_details": {
+                  "properties": {
+                    "action": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "destination_range": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "direction": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "priority": {
+                      "type": "long"
+                    },
+                    "reference": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "source_range": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "source_service_account": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "source_tag": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "target_service_account": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "target_tag": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    }
+                  }
+                }
+              }
+            },
+            "source": {
+              "properties": {
+                "instance": {
+                  "properties": {
+                    "project_id": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "region": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "zone": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    }
+                  }
+                },
+                "vpc": {
+                  "properties": {
+                    "project_id": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "subnetwork_name": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    },
+                    "vpc_name": {
+                      "ignore_above": 1024,
+                      "type": "keyword",
+                      "fields": {
+                        "security": {
+                          "type": "text",
+                          "analyzer": "es_security_analyzer"
+                        }
+                      }
+                    }
+                  }
+                }
+              }
+            },
+            "vpcflow": {
+              "properties": {
+                "reporter": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "security": {
+                      "type": "text",
+                      "analyzer": "es_security_analyzer"
+                    }
+                  }
+                },
+                "rtt": {
+                  "properties": {
+                    "ms": {
+                      "type": "long"
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}