Merge pull request #9329 from Security-Onion-Solutions/hotfix/2.3.190

Hotfix/2.3.190

.github/.gitleaks.toml

@@ -0,0 +1,546 @@
+title = "gitleaks config"
+
+# Gitleaks rules are defined by regular expressions and entropy ranges.
+# Some secrets have unique signatures which make detecting those secrets easy.
+# Examples of those secrets would be GitLab Personal Access Tokens, AWS keys, and GitHub Access Tokens.
+# All these examples have defined prefixes like `glpat`, `AKIA`, `ghp_`, etc.
+#
+# Other secrets might just be a hash which means we need to write more complex rules to verify
+# that what we are matching is a secret.
+#
+# Here is an example of a semi-generic secret
+#
+#   discord_client_secret = "8dyfuiRyq=vVc3RRr_edRk-fK__JItpZ"
+#
+# We can write a regular expression to capture the variable name (identifier),
+# the assignment symbol (like '=' or ':='), and finally the actual secret.
+# The structure of a rule to match this example secret is below:
+#
+#                                                           Beginning string
+#                                                               quotation
+#                                                                   │            End string quotation
+#                                                                   │                      │
+#                                                                   ▼                      ▼
+#    (?i)(discord[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9=_\-]{32})['\"]
+#
+#                   ▲                              ▲                                ▲
+#                   │                              │                                │
+#                   │                              │                                │
+#              identifier                  assignment symbol
+#                                                                                Secret
+#
+[[rules]]
+id = "gitlab-pat"
+description = "GitLab Personal Access Token"
+regex = '''glpat-[0-9a-zA-Z\-\_]{20}'''
+
+[[rules]]
+id = "aws-access-token"
+description = "AWS"
+regex = '''(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}'''
+
+# Cryptographic keys
+[[rules]]
+id = "PKCS8-PK"
+description = "PKCS8 private key"
+regex = '''-----BEGIN PRIVATE KEY-----'''
+
+[[rules]]
+id = "RSA-PK"
+description = "RSA private key"
+regex = '''-----BEGIN RSA PRIVATE KEY-----'''
+
+[[rules]]
+id = "OPENSSH-PK"
+description = "SSH private key"
+regex = '''-----BEGIN OPENSSH PRIVATE KEY-----'''
+
+[[rules]]
+id = "PGP-PK"
+description = "PGP private key"
+regex = '''-----BEGIN PGP PRIVATE KEY BLOCK-----'''
+
+[[rules]]
+id = "github-pat"
+description = "GitHub Personal Access Token"
+regex = '''ghp_[0-9a-zA-Z]{36}'''
+
+[[rules]]
+id = "github-oauth"
+description = "GitHub OAuth Access Token"
+regex = '''gho_[0-9a-zA-Z]{36}'''
+
+[[rules]]
+id = "SSH-DSA-PK"
+description = "SSH (DSA) private key"
+regex = '''-----BEGIN DSA PRIVATE KEY-----'''
+
+[[rules]]
+id = "SSH-EC-PK"
+description = "SSH (EC) private key"
+regex = '''-----BEGIN EC PRIVATE KEY-----'''
+
+
+[[rules]]
+id = "github-app-token"
+description = "GitHub App Token"
+regex = '''(ghu|ghs)_[0-9a-zA-Z]{36}'''
+
+[[rules]]
+id = "github-refresh-token"
+description = "GitHub Refresh Token"
+regex = '''ghr_[0-9a-zA-Z]{76}'''
+
+[[rules]]
+id = "shopify-shared-secret"
+description = "Shopify shared secret"
+regex = '''shpss_[a-fA-F0-9]{32}'''
+
+[[rules]]
+id = "shopify-access-token"
+description = "Shopify access token"
+regex = '''shpat_[a-fA-F0-9]{32}'''
+
+[[rules]]
+id = "shopify-custom-access-token"
+description = "Shopify custom app access token"
+regex = '''shpca_[a-fA-F0-9]{32}'''
+
+[[rules]]
+id = "shopify-private-app-access-token"
+description = "Shopify private app access token"
+regex = '''shppa_[a-fA-F0-9]{32}'''
+
+[[rules]]
+id = "slack-access-token"
+description = "Slack token"
+regex = '''xox[baprs]-([0-9a-zA-Z]{10,48})?'''
+
+[[rules]]
+id = "stripe-access-token"
+description = "Stripe"
+regex = '''(?i)(sk|pk)_(test|live)_[0-9a-z]{10,32}'''
+
+[[rules]]
+id = "pypi-upload-token"
+description = "PyPI upload token"
+regex = '''pypi-AgEIcHlwaS5vcmc[A-Za-z0-9\-_]{50,1000}'''
+
+[[rules]]
+id = "gcp-service-account"
+description = "Google (GCP) Service-account"
+regex = '''\"type\": \"service_account\"'''
+
+[[rules]]
+id = "heroku-api-key"
+description = "Heroku API Key"
+regex = ''' (?i)(heroku[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([0-9A-F]{8}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{12})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "slack-web-hook"
+description = "Slack Webhook"
+regex = '''https://hooks.slack.com/services/T[a-zA-Z0-9_]{8}/B[a-zA-Z0-9_]{8,12}/[a-zA-Z0-9_]{24}'''
+
+[[rules]]
+id = "twilio-api-key"
+description = "Twilio API Key"
+regex = '''SK[0-9a-fA-F]{32}'''
+
+[[rules]]
+id = "age-secret-key"
+description = "Age secret key"
+regex = '''AGE-SECRET-KEY-1[QPZRY9X8GF2TVDW0S3JN54KHCE6MUA7L]{58}'''
+
+[[rules]]
+id = "facebook-token"
+description = "Facebook token"
+regex = '''(?i)(facebook[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-f0-9]{32})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "twitter-token"
+description = "Twitter token"
+regex = '''(?i)(twitter[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-f0-9]{35,44})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "adobe-client-id"
+description = "Adobe Client ID (Oauth Web)"
+regex = '''(?i)(adobe[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-f0-9]{32})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "adobe-client-secret"
+description = "Adobe Client Secret"
+regex = '''(p8e-)(?i)[a-z0-9]{32}'''
+
+[[rules]]
+id = "alibaba-access-key-id"
+description = "Alibaba AccessKey ID"
+regex = '''(LTAI)(?i)[a-z0-9]{20}'''
+
+[[rules]]
+id = "alibaba-secret-key"
+description = "Alibaba Secret Key"
+regex = '''(?i)(alibaba[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{30})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "asana-client-id"
+description = "Asana Client ID"
+regex = '''(?i)(asana[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([0-9]{16})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "asana-client-secret"
+description = "Asana Client Secret"
+regex = '''(?i)(asana[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{32})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "atlassian-api-token"
+description = "Atlassian API token"
+regex = '''(?i)(atlassian[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{24})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "bitbucket-client-id"
+description = "Bitbucket client ID"
+regex = '''(?i)(bitbucket[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{32})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "bitbucket-client-secret"
+description = "Bitbucket client secret"
+regex = '''(?i)(bitbucket[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9_\-]{64})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "beamer-api-token"
+description = "Beamer API token"
+regex = '''(?i)(beamer[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](b_[a-z0-9=_\-]{44})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "clojars-api-token"
+description = "Clojars API token"
+regex = '''(CLOJARS_)(?i)[a-z0-9]{60}'''
+
+[[rules]]
+id = "contentful-delivery-api-token"
+description = "Contentful delivery API token"
+regex = '''(?i)(contentful[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9\-=_]{43})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "databricks-api-token"
+description = "Databricks API token"
+regex = '''dapi[a-h0-9]{32}'''
+
+[[rules]]
+id = "discord-api-token"
+description = "Discord API key"
+regex = '''(?i)(discord[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-h0-9]{64})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "discord-client-id"
+description = "Discord client ID"
+regex = '''(?i)(discord[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([0-9]{18})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "discord-client-secret"
+description = "Discord client secret"
+regex = '''(?i)(discord[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9=_\-]{32})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "doppler-api-token"
+description = "Doppler API token"
+regex = '''['\"](dp\.pt\.)(?i)[a-z0-9]{43}['\"]'''
+
+[[rules]]
+id = "dropbox-api-secret"
+description = "Dropbox API secret/key"
+regex = '''(?i)(dropbox[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{15})['\"]'''
+
+[[rules]]
+id = "dropbox--api-key"
+description = "Dropbox API secret/key"
+regex = '''(?i)(dropbox[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{15})['\"]'''
+
+[[rules]]
+id = "dropbox-short-lived-api-token"
+description = "Dropbox short lived API token"
+regex = '''(?i)(dropbox[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](sl\.[a-z0-9\-=_]{135})['\"]'''
+
+[[rules]]
+id = "dropbox-long-lived-api-token"
+description = "Dropbox long lived API token"
+regex = '''(?i)(dropbox[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"][a-z0-9]{11}(AAAAAAAAAA)[a-z0-9\-_=]{43}['\"]'''
+
+[[rules]]
+id = "duffel-api-token"
+description = "Duffel API token"
+regex = '''['\"]duffel_(test|live)_(?i)[a-z0-9_-]{43}['\"]'''
+
+[[rules]]
+id = "dynatrace-api-token"
+description = "Dynatrace API token"
+regex = '''['\"]dt0c01\.(?i)[a-z0-9]{24}\.[a-z0-9]{64}['\"]'''
+
+[[rules]]
+id = "easypost-api-token"
+description = "EasyPost API token"
+regex = '''['\"]EZAK(?i)[a-z0-9]{54}['\"]'''
+
+[[rules]]
+id = "easypost-test-api-token"
+description = "EasyPost test API token"
+regex = '''['\"]EZTK(?i)[a-z0-9]{54}['\"]'''
+
+[[rules]]
+id = "fastly-api-token"
+description = "Fastly API token"
+regex = '''(?i)(fastly[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9\-=_]{32})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "finicity-client-secret"
+description = "Finicity client secret"
+regex = '''(?i)(finicity[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{20})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "finicity-api-token"
+description = "Finicity API token"
+regex = '''(?i)(finicity[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-f0-9]{32})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "flutterwave-public-key"
+description = "Flutterwave public key"
+regex = '''FLWPUBK_TEST-(?i)[a-h0-9]{32}-X'''
+
+[[rules]]
+id = "flutterwave-secret-key"
+description = "Flutterwave secret key"
+regex = '''FLWSECK_TEST-(?i)[a-h0-9]{32}-X'''
+
+[[rules]]
+id = "flutterwave-enc-key"
+description = "Flutterwave encrypted key"
+regex = '''FLWSECK_TEST[a-h0-9]{12}'''
+
+[[rules]]
+id = "frameio-api-token"
+description = "Frame.io API token"
+regex = '''fio-u-(?i)[a-z0-9\-_=]{64}'''
+
+[[rules]]
+id = "gocardless-api-token"
+description = "GoCardless API token"
+regex = '''['\"]live_(?i)[a-z0-9\-_=]{40}['\"]'''
+
+[[rules]]
+id = "grafana-api-token"
+description = "Grafana API token"
+regex = '''['\"]eyJrIjoi(?i)[a-z0-9\-_=]{72,92}['\"]'''
+
+[[rules]]
+id = "hashicorp-tf-api-token"
+description = "HashiCorp Terraform user/org API token"
+regex = '''['\"](?i)[a-z0-9]{14}\.atlasv1\.[a-z0-9\-_=]{60,70}['\"]'''
+
+[[rules]]
+id = "hubspot-api-token"
+description = "HubSpot API token"
+regex = '''(?i)(hubspot[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-h0-9]{8}-[a-h0-9]{4}-[a-h0-9]{4}-[a-h0-9]{4}-[a-h0-9]{12})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "intercom-api-token"
+description = "Intercom API token"
+regex = '''(?i)(intercom[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9=_]{60})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "intercom-client-secret"
+description = "Intercom client secret/ID"
+regex = '''(?i)(intercom[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-h0-9]{8}-[a-h0-9]{4}-[a-h0-9]{4}-[a-h0-9]{4}-[a-h0-9]{12})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "ionic-api-token"
+description = "Ionic API token"
+regex = '''(?i)(ionic[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](ion_[a-z0-9]{42})['\"]'''
+
+[[rules]]
+id = "linear-api-token"
+description = "Linear API token"
+regex = '''lin_api_(?i)[a-z0-9]{40}'''
+
+[[rules]]
+id = "linear-client-secret"
+description = "Linear client secret/ID"
+regex = '''(?i)(linear[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-f0-9]{32})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "lob-api-key"
+description = "Lob API Key"
+regex = '''(?i)(lob[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]((live|test)_[a-f0-9]{35})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "lob-pub-api-key"
+description = "Lob Publishable API Key"
+regex = '''(?i)(lob[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]((test|live)_pub_[a-f0-9]{31})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "mailchimp-api-key"
+description = "Mailchimp API key"
+regex = '''(?i)(mailchimp[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-f0-9]{32}-us20)['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "mailgun-private-api-token"
+description = "Mailgun private API token"
+regex = '''(?i)(mailgun[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](key-[a-f0-9]{32})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "mailgun-pub-key"
+description = "Mailgun public validation key"
+regex = '''(?i)(mailgun[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](pubkey-[a-f0-9]{32})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "mailgun-signing-key"
+description = "Mailgun webhook signing key"
+regex = '''(?i)(mailgun[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-h0-9]{32}-[a-h0-9]{8}-[a-h0-9]{8})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "mapbox-api-token"
+description = "Mapbox API token"
+regex = '''(?i)(pk\.[a-z0-9]{60}\.[a-z0-9]{22})'''
+
+[[rules]]
+id = "messagebird-api-token"
+description = "MessageBird API token"
+regex = '''(?i)(messagebird[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{25})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "messagebird-client-id"
+description = "MessageBird API client ID"
+regex = '''(?i)(messagebird[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-h0-9]{8}-[a-h0-9]{4}-[a-h0-9]{4}-[a-h0-9]{4}-[a-h0-9]{12})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "new-relic-user-api-key"
+description = "New Relic user API Key"
+regex = '''['\"](NRAK-[A-Z0-9]{27})['\"]'''
+
+[[rules]]
+id = "new-relic-user-api-id"
+description = "New Relic user API ID"
+regex = '''(?i)(newrelic[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([A-Z0-9]{64})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "new-relic-browser-api-token"
+description = "New Relic ingest browser API token"
+regex = '''['\"](NRJS-[a-f0-9]{19})['\"]'''
+
+[[rules]]
+id = "npm-access-token"
+description = "npm access token"
+regex = '''['\"](npm_(?i)[a-z0-9]{36})['\"]'''
+
+[[rules]]
+id = "planetscale-password"
+description = "PlanetScale password"
+regex = '''pscale_pw_(?i)[a-z0-9\-_\.]{43}'''
+
+[[rules]]
+id = "planetscale-api-token"
+description = "PlanetScale API token"
+regex = '''pscale_tkn_(?i)[a-z0-9\-_\.]{43}'''
+
+[[rules]]
+id = "postman-api-token"
+description = "Postman API token"
+regex = '''PMAK-(?i)[a-f0-9]{24}\-[a-f0-9]{34}'''
+
+[[rules]]
+id = "pulumi-api-token"
+description = "Pulumi API token"
+regex = '''pul-[a-f0-9]{40}'''
+
+[[rules]]
+id = "rubygems-api-token"
+description = "Rubygem API token"
+regex = '''rubygems_[a-f0-9]{48}'''
+
+[[rules]]
+id = "sendgrid-api-token"
+description = "SendGrid API token"
+regex = '''SG\.(?i)[a-z0-9_\-\.]{66}'''
+
+[[rules]]
+id = "sendinblue-api-token"
+description = "Sendinblue API token"
+regex = '''xkeysib-[a-f0-9]{64}\-(?i)[a-z0-9]{16}'''
+
+[[rules]]
+id = "shippo-api-token"
+description = "Shippo API token"
+regex = '''shippo_(live|test)_[a-f0-9]{40}'''
+
+[[rules]]
+id = "linkedin-client-secret"
+description = "LinkedIn Client secret"
+regex = '''(?i)(linkedin[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z]{16})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "linkedin-client-id"
+description = "LinkedIn Client ID"
+regex = '''(?i)(linkedin[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{14})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "twitch-api-token"
+description = "Twitch API token"
+regex = '''(?i)(twitch[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{30})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "typeform-api-token"
+description = "Typeform API token"
+regex = '''(?i)(typeform[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}(tfp_[a-z0-9\-_\.=]{59})'''
+secretGroup = 3
+
+[[rules]]
+id = "generic-api-key"
+description = "Generic API Key"
+regex = '''(?i)((key|api[^Version]|token|secret|password)[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([0-9a-zA-Z\-_=]{8,64})['\"]'''
+entropy = 3.7
+secretGroup = 4
+
+
+[allowlist]
+description = "global allow lists"
+regexes = ['''219-09-9999''', '''078-05-1120''', '''(9[0-9]{2}|666)-\d{2}-\d{4}''']
+paths = [
+    '''gitleaks.toml''',
+    '''(.*?)(jpg|gif|doc|pdf|bin|svg|socket)$''',
+    '''(go.mod|go.sum)$''',
+    
+    '''salt/nginx/files/enterprise-attack.json'''
+]

.github/workflows/contrib.yml

@@ -0,0 +1,24 @@
+name: contrib
+on:
+  issue_comment:
+    types: [created]
+  pull_request_target:
+    types: [opened,closed,synchronize]
+
+jobs:
+  CLAssistant:
+    runs-on: ubuntu-latest
+    steps:
+      - name: "Contributor Check"
+        if: (github.event.comment.body == 'recheck' || github.event.comment.body == 'I have read the CLA Document and I hereby sign the CLA') || github.event_name == 'pull_request_target'
+        uses: cla-assistant/github-action@v2.1.3-beta
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          PERSONAL_ACCESS_TOKEN : ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+        with:
+          path-to-signatures: 'signatures_v1.json'
+          path-to-document: 'https://securityonionsolutions.com/cla' 
+          allowlist: dependabot[bot],jertel,dougburks,TOoSmOotH,weslambert,defensivedepth,m0duspwnens
+          remote-organization-name: Security-Onion-Solutions
+          remote-repository-name: licensing
+

.github/workflows/leaktest.yml

@@ -12,4 +12,6 @@ jobs:
         fetch-depth: '0'
 
     - name: Gitleaks
-      uses: zricethezav/gitleaks-action@master
+      uses: gitleaks/gitleaks-action@v1.6.0
+      with:
+        config-path: .github/.gitleaks.toml

.github/workflows/pythontest.yml

@@ -0,0 +1,31 @@
+name: python-test
+
+on: [push, pull_request]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10"]
+        python-code-path: ["salt/sensoroni/files/analyzers"]
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install flake8 pytest pytest-cov
+        find . -name requirements.txt -exec pip install -r {} \;
+    - name: Lint with flake8
+      run: |
+        flake8 ${{ matrix.python-code-path }} --show-source --max-complexity=12  --doctests --max-line-length=200 --statistics
+    - name: Test with pytest
+      run: |
+        pytest ${{ matrix.python-code-path }} --cov=${{ matrix.python-code-path }} --doctest-modules --cov-report=term --cov-fail-under=100 --cov-config=${{ matrix.python-code-path }}/pytest.ini

.gitignore

@@ -56,4 +56,15 @@ $RECYCLE.BIN/
 # Windows shortcuts
 *.lnk
 
-# End of https://www.gitignore.io/api/macos,windows
+# End of https://www.gitignore.io/api/macos,windows
+
+# Pytest output
+__pycache__
+.pytest_cache
+.coverage
+*.pyc
+.venv
+
+# Analyzer dev/test config files
+*_dev.yaml
+site-packages

HOTFIX

@@ -1 +1 @@
-04012022 04052022 04072022
+20221207

README.md

@@ -1,14 +1,20 @@
-## Security Onion 2.3.110
+## Security Onion 2.3
 
-Security Onion 2.3.110 is here!
+Security Onion 2.3 is here!
 
 ## Screenshots
 
 Alerts
-![Alerts](./assets/images/screenshots/alerts-1.png)
+![Alerts](./assets/images/screenshots/alerts.png)
+
+Dashboards
+![Dashboards](./assets/images/screenshots/dashboards.png)
 
 Hunt
-![Hunt](./assets/images/screenshots/hunt-1.png)
+![Hunt](./assets/images/screenshots/hunt.png)
+
+Cases
+![Cases](./assets/images/screenshots/cases-comments.png)
 
 ### Release Notes
 

VERIFY_ISO.md

@@ -1,18 +1,18 @@
-### 2.3.110-20220407 ISO image built on 2022/04/07
+### 2.3.190-20221207 ISO image built on 2022/12/07
 
 
 
 ### Download and Verify
 
-2.3.110-20220407 ISO image:  
-https://download.securityonion.net/file/securityonion/securityonion-2.3.110-20220407.iso
+2.3.190-20221207 ISO image:  
+https://download.securityonion.net/file/securityonion/securityonion-2.3.190-20221207.iso
 
-MD5: 928D589709731EFE9942CA134A6F4C6B  
-SHA1: CA588A684586CC0D5BDE5E0E41C935FFB939B6C7  
-SHA256: CBF8743838AF2C7323E629FB6B28D5DD00AE6658B0E29E4D0916411D2D526BD2 
+MD5: F7F222325A5C1C880E11B667FEE913CA  
+SHA1: F7DFE818A0CED391548CDF0DE3B4D2A24E16A532  
+SHA256: 95E62E0D347A80C8A9CD4979D6F6BE8B302A12424A888410025E9AAB8BD504B2 
 
 Signature for ISO image:  
-https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.110-20220407.iso.sig
+https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.190-20221207.iso.sig
 
 Signing key:  
 https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS  
@@ -26,22 +26,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/ma
 
 Download the signature file for the ISO:  
 ```
-wget https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.110-20220407.iso.sig
+wget https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.190-20221207.iso.sig
 ```
 
 Download the ISO image:  
 ```
-wget https://download.securityonion.net/file/securityonion/securityonion-2.3.110-20220407.iso
+wget https://download.securityonion.net/file/securityonion/securityonion-2.3.190-20221207.iso
 ```
 
 Verify the downloaded ISO image using the signature file:  
 ```
-gpg --verify securityonion-2.3.110-20220407.iso.sig securityonion-2.3.110-20220407.iso
+gpg --verify securityonion-2.3.190-20221207.iso.sig securityonion-2.3.190-20221207.iso
 ```
 
 The output should show "Good signature" and the Primary key fingerprint should match what's shown below:
 ```
-gpg: Signature made Thu 07 Apr 2022 03:30:03 PM EDT using RSA key ID FE507013
+gpg: Signature made Wed 07 Dec 2022 02:36:23 PM EST using RSA key ID FE507013
 gpg: Good signature from "Security Onion Solutions, LLC <info@securityonionsolutions.com>"
 gpg: WARNING: This key is not certified with a trusted signature!
 gpg:          There is no indication that the signature belongs to the owner.

VERSION

@@ -1 +1 @@
-2.3.110
+2.3.190

pillar/elasticsearch/index_templates.sls

@@ -0,0 +1,2 @@
+elasticsearch:
+  index_settings:

pillar/logstash/nodes.sls

@@ -2,7 +2,7 @@
 {% set cached_grains = salt.saltutil.runner('cache.grains', tgt='*') %}
 {% for minionid, ip in salt.saltutil.runner(
     'mine.get',
-    tgt='G@role:so-manager or G@role:so-managersearch or G@role:so-standalone or G@role:so-node or G@role:so-heavynode or G@role:so-receiver or G@role:so-helix ',
+    tgt='G@role:so-manager or G@role:so-managersearch or G@role:so-standalone or G@role:so-node or G@role:so-heavynode or G@role:so-receiver or G@role:so-helix',
     fun='network.ip_addrs',
     tgt_type='compound') | dictsort()
 %}

pillar/logstash/search.sls

@@ -13,4 +13,6 @@ logstash:
         - so/9600_output_ossec.conf.jinja
         - so/9700_output_strelka.conf.jinja
         - so/9800_output_logscan.conf.jinja
+        - so/9801_output_rita.conf.jinja
+        - so/9802_output_kratos.conf.jinja
         - so/9900_output_endgame.conf.jinja

pillar/top.sls

@@ -15,12 +15,12 @@ base:
     - logstash
     - logstash.manager
     - logstash.search
-    - elasticsearch.search
+    - elasticsearch.index_templates
 
   '*_manager':
     - logstash
     - logstash.manager
-    - elasticsearch.manager
+    - elasticsearch.index_templates
 
   '*_manager or *_managersearch':
     - match: compound
@@ -46,7 +46,7 @@ base:
     - zeeklogs
     - secrets
     - healthcheck.eval
-    - elasticsearch.eval
+    - elasticsearch.index_templates
 {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
     - elasticsearch.auth
 {% endif %}
@@ -60,7 +60,7 @@ base:
     - logstash
     - logstash.manager
     - logstash.search
-    - elasticsearch.search
+    - elasticsearch.index_templates
 {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
     - elasticsearch.auth
 {% endif %}
@@ -106,7 +106,7 @@ base:
   '*_searchnode':
     - logstash
     - logstash.search
-    - elasticsearch.search
+    - elasticsearch.index_templates
     - elasticsearch.auth
     - global
     - minions.{{ grains.id }}
@@ -122,7 +122,7 @@ base:
   '*_import':
     - zeeklogs
     - secrets
-    - elasticsearch.eval
+    - elasticsearch.index_templates
 {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
     - elasticsearch.auth
 {% endif %}
@@ -131,3 +131,6 @@ base:
 {% endif %}
     - global
     - minions.{{ grains.id }}
+
+  '*_workstation':
+    - minions.{{ grains.id }}

pillar/zeek/init.sls

@@ -48,6 +48,19 @@ zeek:
       - securityonion/bpfconf
       - securityonion/communityid
       - securityonion/file-extraction
+      - oui-logging
+      - icsnpp-modbus
+      - icsnpp-dnp3
+      - icsnpp-bacnet
+      - icsnpp-ethercat
+      - icsnpp-enip
+      - icsnpp-opcua-binary
+      - icsnpp-bsap
+      - icsnpp-s7comm
+      - zeek-plugin-tds
+      - zeek-plugin-profinet
+      - zeek-spicy-wireguard
+      - zeek-spicy-stun
     '@load-sigs':
       - frameworks/signatures/detect-windows-shells
     redef:

salt/allowed_states.map.jinja

@@ -1,6 +1,5 @@
 {% set ZEEKVER = salt['pillar.get']('global:mdengine', '') %}
 {% set WAZUH = salt['pillar.get']('global:wazuh', '0') %}
-{% set THEHIVE = salt['pillar.get']('manager:thehive', '0') %}
 {% set PLAYBOOK = salt['pillar.get']('manager:playbook', '0') %}
 {% set FREQSERVER = salt['pillar.get']('manager:freq', '0') %}
 {% set DOMAINSTATS = salt['pillar.get']('manager:domainstats', '0') %}
@@ -218,6 +217,8 @@
           'schedule',
           'docker_clean'
           ],
+      'so-workstation': [
+          ],
   }, grain='role') %}
 
   {% if FILEBEAT and grains.role in ['so-helixsensor', 'so-eval', 'so-manager', 'so-standalone', 'so-node', 'so-managersearch', 'so-heavynode', 'so-import', 'so-receiver'] %}
@@ -273,10 +274,6 @@
     {% do allowed_states.append('elastalert') %}
   {% endif %}
 
-  {% if (THEHIVE != 0) and grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-managersearch'] %}
-    {% do allowed_states.append('thehive') %}
-  {% endif %}
-
   {% if (PLAYBOOK !=0) and grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-managersearch'] %}
     {% do allowed_states.append('playbook') %}
   {% endif %}

salt/common/files/sensor-rotate.conf

@@ -19,4 +19,17 @@
     extension .log
     dateext
     dateyesterday
-}
+}
+
+/opt/so/log/strelka/filecheck.log
+{
+    daily
+    rotate 14
+    missingok
+    copytruncate
+    compress
+    create
+    extension .log
+    dateext
+    dateyesterday
+}

salt/common/init.sls

@@ -38,15 +38,15 @@ socore:
 soconfperms:
   file.directory:
     - name: /opt/so/conf
-    - uid: 939
-    - gid: 939
+    - user: 939
+    - group: 939
     - dir_mode: 770
 
 sostatusconf:
   file.directory:
     - name: /opt/so/conf/so-status
-    - uid: 939
-    - gid: 939
+    - user: 939
+    - group: 939
     - dir_mode: 770
 
 so-status.conf:
@@ -57,8 +57,8 @@ so-status.conf:
 sosaltstackperms:
   file.directory:
     - name: /opt/so/saltstack
-    - uid: 939
-    - gid: 939
+    - user: 939
+    - group: 939
     - dir_mode: 770
 
 so_log_perms:
@@ -300,8 +300,17 @@ sostatus_log:
     - month: '*'
     - dayweek: '*'
 
-
 {% if role in ['eval', 'manager', 'managersearch', 'standalone'] %}
+# Install cron job to determine size of influxdb for telegraf
+'du -s -k /nsm/influxdb | cut -f1 > /opt/so/log/telegraf/influxdb_size.log 2>&1':
+  cron.present:
+    - user: root
+    - minute: '*/1'
+    - hour: '*'
+    - daymonth: '*'
+    - month: '*'
+    - dayweek: '*'
+    
 # Lock permissions on the backup directory
 backupdir:
   file.directory:

salt/common/tools/sbin/so-analyst-install

@@ -15,295 +15,86 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-if [ "$(id -u)" -ne 0 ]; then
-	echo "This script must be run using sudo!"
-	exit 1
-fi
+doc_workstation_url="https://docs.securityonion.net/en/2.3/analyst-vm.html"
+{# we only want the script to install the workstation if it is CentOS -#}
+{% if grains.os == 'CentOS' -%}
+{#   if this is a manager -#}
+{%   if grains.master == grains.id.split('_')|first -%}
 
-INSTALL_LOG=/root/so-analyst-install.log
-exec &> >(tee -a "$INSTALL_LOG")
+source /usr/sbin/so-common
+pillar_file="/opt/so/saltstack/local/pillar/minions/{{grains.id}}.sls"
 
-log() {
-	msg=$1
-	level=${2:-I}
-	now=$(TZ=GMT date +"%Y-%m-%dT%H:%M:%SZ")
-	echo -e "$now | $level | $msg" >> "$INSTALL_LOG" 2>&1
-}
+if [ -f "$pillar_file" ]; then
+  if ! grep -q "^workstation:$" "$pillar_file"; then
 
-error() {
-	log "$1" "E"
-}
-
-info() {
-	log "$1" "I"
-}
-
-title() {
-	echo -e "\n-----------------------------\n $1\n-----------------------------\n" >> "$INSTALL_LOG" 2>&1
-}
-
-logCmd() {
-	cmd=$1
-	info "Executing command: $cmd"
-	$cmd >> "$INSTALL_LOG" 2>&1
-}
-
-analyze_system() {
-	title "System Characteristics"
-	logCmd "uptime"
-	logCmd "uname -a"
-	logCmd "free -h"
-	logCmd "lscpu"
-	logCmd "df -h"
-	logCmd "ip a"
-}
-
-analyze_system
-
-OS=$(grep PRETTY_NAME /etc/os-release | grep 'CentOS Linux 7')
-if [ $? -ne 0 ]; then
-  echo "This is an unsupported OS. Please use CentOS 7 to install the analyst node."
-  exit 1
-fi
-
-if [[ "$manufacturer" == "Security Onion Solutions" && "$family" == "Automated" ]]; then
-  INSTALL=yes
-  CURLCONTINUE=no
-else
-  INSTALL=''
-  CURLCONTINUE=''
-fi
-
-FIRSTPASS=yes
-while [[ $INSTALL != "yes" ]] && [[ $INSTALL != "no" ]]; do
-  if [[ "$FIRSTPASS" == "yes" ]]; then
-    clear
-    echo "###########################################"
-    echo "##          ** W A R N I N G **          ##"
-    echo "##    _______________________________    ##"
-    echo "##                                       ##"
-    echo "##    Installing the Security Onion      ##"
-    echo "##   analyst node on this device will    ##"
-    echo "##       make permanent changes to       ##"
-    echo "##              the system.              ##"
-    echo "##                                       ##"
-    echo "###########################################"
-    echo "Do you wish to continue? (Type the entire word 'yes' to proceed or 'no' to exit)"
-    FIRSTPASS=no
-  else
-    echo "Please type 'yes' to continue or 'no' to exit."
-  fi      
-  read INSTALL
-done
-
-if [[ $INSTALL == "no" ]]; then
-  echo "Exiting analyst node installation."
-  exit 0
-fi
-
-echo "Testing for internet connection with curl https://securityonionsolutions.com/"
-CANCURL=$(curl -sI https://securityonionsolutions.com/ | grep "200 OK")
-  if [ $? -ne 0 ]; then
     FIRSTPASS=yes
-    while [[ $CURLCONTINUE != "yes" ]] && [[ $CURLCONTINUE != "no" ]]; do
+    while [[ $INSTALL != "yes" ]] && [[ $INSTALL != "no" ]]; do
       if [[ "$FIRSTPASS" == "yes" ]]; then
-        echo "We could not access https://securityonionsolutions.com/."
-        echo "Since packages are downloaded from the internet, internet access is required."
-        echo "If you would like to ignore this warning and continue anyway, please type 'yes'."
-        echo "Otherwise, type 'no' to exit."
+        echo "###########################################"
+        echo "##          ** W A R N I N G **          ##"
+        echo "##    _______________________________    ##"
+        echo "##                                       ##"
+        echo "##    Installing the Security Onion      ##"
+        echo "##   analyst node on this device will    ##"
+        echo "##       make permanent changes to       ##"
+        echo "##              the system.              ##"
+        echo "##    A system reboot will be required   ##"
+        echo "##        to complete the install.       ##"
+        echo "##                                       ##"
+        echo "###########################################"
+        echo "Do you wish to continue? (Type the entire word 'yes' to proceed or 'no' to exit)"
         FIRSTPASS=no
       else
         echo "Please type 'yes' to continue or 'no' to exit."
-      fi  
-      read CURLCONTINUE
+      fi      
+      read INSTALL
     done
-    if [[ "$CURLCONTINUE" == "no" ]]; then
+
+    if [[ $INSTALL == "no" ]]; then
       echo "Exiting analyst node installation."
       exit 0
     fi
-  else
-    echo "We were able to curl https://securityonionsolutions.com/."
-    sleep 3
+
+    # Add workstation pillar to the minion's pillar file
+    printf '%s\n'\
+      "workstation:"\
+      "  gui:"\
+      "    enabled: true"\
+		  "" >> "$pillar_file"
+    echo "Applying the workstation state. This could take some time since there are many packages that need to be installed."
+    if salt-call state.apply workstation -linfo queue=True; then # make sure the state ran successfully
+      echo ""
+      echo "Analyst workstation has been installed!"
+      echo "Press ENTER to reboot or Ctrl-C to cancel."
+      read pause
+
+      reboot;
+    else
+      echo "There was an issue applying the workstation state. Please review the log above or at /opt/so/logs/salt/minion."
+    fi
+  else # workstation is already added
+    echo "The workstation pillar already exists in $pillar_file."
+    echo "To enable/disable the gui, set 'workstation:gui:enabled' to true or false in $pillar_file."
+    echo "Additional documentation can be found at $doc_workstation_url."
   fi
-
-# Install a GUI text editor
-yum -y install gedit
-
-# Install misc utils
-yum -y install wget curl unzip epel-release yum-plugin-versionlock;
-
-# Install xWindows
-yum -y groupinstall "X Window System";
-yum -y install gnome-classic-session gnome-terminal nautilus-open-terminal control-center liberation-mono-fonts;
-unlink /etc/systemd/system/default.target;
-ln -sf /lib/systemd/system/graphical.target /etc/systemd/system/default.target;
-yum -y install file-roller
-
-# Install Mono - prereq for NetworkMiner
-yum -y install mono-core mono-basic mono-winforms expect
-
-# Install NetworkMiner
-yum -y install libcanberra-gtk2;
-wget https://www.netresec.com/?download=NetworkMiner -O /tmp/nm.zip;
-mkdir -p /opt/networkminer/
-unzip /tmp/nm.zip -d /opt/networkminer/;
-rm /tmp/nm.zip;
-mv /opt/networkminer/NetworkMiner_*/* /opt/networkminer/
-chmod +x /opt/networkminer/NetworkMiner.exe;
-chmod -R go+w /opt/networkminer/AssembledFiles/;
-chmod -R go+w /opt/networkminer/Captures/;
-# Create networkminer shim
-cat << EOF >> /bin/networkminer
-#!/bin/bash
-/bin/mono /opt/networkminer/NetworkMiner.exe --noupdatecheck "\$@"
-EOF
-chmod +x /bin/networkminer
-# Convert networkminer ico file to png format
-yum -y install ImageMagick
-convert /opt/networkminer/networkminericon.ico /opt/networkminer/networkminericon.png
-# Create menu entry
-cat << EOF >> /usr/share/applications/networkminer.desktop
-[Desktop Entry]
-Name=NetworkMiner
-Comment=NetworkMiner
-Encoding=UTF-8
-Exec=/bin/networkminer %f
-Icon=/opt/networkminer/networkminericon-4.png
-StartupNotify=true
-Terminal=false
-X-MultipleArgs=false
-Type=Application
-MimeType=application/x-pcap;
-Categories=Network;
-EOF
-
-# Set default monospace font to Liberation
-cat << EOF >> /etc/fonts/local.conf
- <match target="pattern">
-  <test name="family" qual="any">
-   <string>monospace</string>
-  </test>
-  <edit binding="strong" mode="prepend" name="family">
-   <string>Liberation Mono</string>
-  </edit>
- </match>
-EOF
-
-# Install Wireshark for Gnome
-yum -y install wireshark-gnome; 
-
-# Install dnsiff
-yum -y install dsniff;
-
-# Install hping3
-yum -y install hping3;
-
-# Install netsed
-yum -y install netsed;
-
-# Install ngrep
-yum -y install ngrep;
-
-# Install scapy
-yum -y install python36-scapy;
-
-# Install ssldump
-yum -y install ssldump;
-
-# Install tcpdump
-yum -y install tcpdump;
-
-# Install tcpflow
-yum -y install tcpflow;
-
-# Install tcpxtract
-yum -y install tcpxtract;
-
-# Install whois
-yum -y install whois;
-
-# Install foremost
-yum -y install https://forensics.cert.org/centos/cert/7/x86_64//foremost-1.5.7-13.1.el7.x86_64.rpm;
-
-# Install chromium
-yum -y install chromium;
-
-# Install tcpstat
-yum -y install https://github.com/Security-Onion-Solutions/securityonion-docker-rpm/releases/download/securityonion-tcpstat-1.5.0/securityonion-tcpstat-1.5.0.rpm;
-
-# Install tcptrace
-yum -y install https://github.com/Security-Onion-Solutions/securityonion-docker-rpm/releases/download/securityonion-tcptrace-6.6.7/securityonion-tcptrace-6.6.7.rpm;
-
-# Install sslsplit
-yum -y install libevent;
-yum -y install sslsplit;
-
-# Install Bit-Twist
-yum -y install https://github.com/Security-Onion-Solutions/securityonion-docker-rpm/releases/download/securityonion-bittwist-2.0.0/securityonion-bittwist-2.0.0.rpm;
-
-# Install chaosreader
-yum -y install perl-IO-Compress perl-Net-DNS;
-yum -y install https://github.com/Security-Onion-Solutions/securityonion-docker-rpm/releases/download/securityonion-chaosreader-0.95.10/securityonion-chaosreader-0.95.10.rpm;
-chmod +x /bin/chaosreader;
-
-if [ -f ../../files/analyst/README ]; then
-  cp ../../files/analyst/README /;
-  cp ../../files/analyst/so-wallpaper.jpg /usr/share/backgrounds/;
-  cp ../../files/analyst/so-lockscreen.jpg /usr/share/backgrounds/;
-  cp ../../files/analyst/so-login-logo-dark.svg /usr/share/pixmaps/;
-else
-  cp /opt/so/saltstack/default/salt/common/files/analyst/README /;
-  cp /opt/so/saltstack/default/salt/common/files/analyst/so-wallpaper.jpg /usr/share/backgrounds/;
-  cp /opt/so/saltstack/default/salt/common/files/analyst/so-lockscreen.jpg /usr/share/backgrounds/;
-  cp /opt/so/saltstack/default/salt/common/files/analyst/so-login-logo-dark.svg /usr/share/pixmaps/;
+else # if the pillar file doesn't exist
+  echo "Could not find $pillar_file and add the workstation pillar."
 fi
 
-# Set background wallpaper
-cat << EOF >> /etc/dconf/db/local.d/00-background
-# Specify the dconf path
-[org/gnome/desktop/background]
+{#-  if this is not a manager #}
+{%   else -%}
 
-# Specify the path to the desktop background image file
-picture-uri='file:///usr/share/backgrounds/so-wallpaper.jpg'
-# Specify one of the rendering options for the background image:
-# 'none', 'wallpaper', 'centered', 'scaled', 'stretched', 'zoom', 'spanned'
-picture-options='zoom'
-# Specify the left or top color when drawing gradients or the solid color
-primary-color='000000'
-# Specify the right or bottom color when drawing gradients
-secondary-color='FFFFFF'
-EOF
+echo "Since this is not a manager, the pillar values to enable analyst workstation must be set manually. Please view the documentation at $doc_workstation_url."
 
-# Set lock screen
-cat << EOF >> /etc/dconf/db/local.d/00-screensaver
-[org/gnome/desktop/session]
-idle-delay=uint32 180
+{#- endif if this is a manager #}
+{%   endif -%}
 
-[org/gnome/desktop/screensaver]
-lock-enabled=true
-lock-delay=uint32 120
-picture-options='zoom'
-picture-uri='file:///usr/share/backgrounds/so-lockscreen.jpg'
-EOF
+{#- if not CentOS #}
+{%- else %}
 
-cat << EOF >> /etc/dconf/db/local.d/locks/screensaver
-/org/gnome/desktop/session/idle-delay
-/org/gnome/desktop/screensaver/lock-enabled
-/org/gnome/desktop/screensaver/lock-delay
-EOF
+echo "The Analyst Workstation can only be installed on CentOS. Please view the documentation at $doc_workstation_url."
 
-# Do not show the user list at login screen
-cat << EOF >> /etc/dconf/db/local.d/00-login-screen
-[org/gnome/login-screen]
-logo='/usr/share/pixmaps/so-login-logo-dark.svg'
-disable-user-list=true
-EOF
+{#- endif grains.os == CentOS #}
+{% endif -%}
 
-dconf update;
-
-echo
-echo "Analyst workstation has been installed!"
-echo "Press ENTER to reboot or Ctrl-C to cancel."
-read pause
-
-reboot;
+exit 0

salt/common/tools/sbin/so-bpf-compile

@@ -29,7 +29,7 @@ fi
 
 interface="$1"
 shift
-sudo tcpdump -i $interface -ddd $@ | tail -n+2 |
+tcpdump -i $interface -ddd $@ | tail -n+2 |
 while read line; do
   cols=( $line )
   printf "%04x%02x%02x%08x" ${cols[0]} ${cols[1]} ${cols[2]} ${cols[3]}

salt/common/tools/sbin/so-common

@@ -120,6 +120,30 @@ check_elastic_license() {
 	fi  
 }
 
+check_salt_master_status() {
+	local timeout=$1
+	echo "Checking if we can talk to the salt master"
+	salt-call state.show_top concurrent=true
+
+	return
+}
+
+check_salt_minion_status() {
+	local timeout=$1
+	echo "Checking if the salt minion will respond to jobs" >> "$setup_log" 2>&1
+	salt "$MINION_ID" test.ping -t $timeout > /dev/null 2>&1
+	local status=$?
+	if [ $status -gt 0 ]; then
+		echo "  Minion did not respond" >> "$setup_log" 2>&1
+	else
+		echo "  Received job response from salt minion" >> "$setup_log" 2>&1
+	fi
+
+	return $status
+}
+
+
+
 copy_new_files() {
   # Copy new files over to the salt dir
   cd $UPDATE_DIR
@@ -367,6 +391,7 @@ run_check_net_err() {
 		exit $exit_code
 	fi
 }
+
 set_cron_service_name() {
   if [[ "$OS" == "centos" ]]; then
     cron_service_name="crond"

salt/common/tools/sbin/so-cortex-restart

@@ -17,5 +17,4 @@
 
 . /usr/sbin/so-common
 
-/usr/sbin/so-stop cortex $1
-/usr/sbin/so-start thehive $1
+echo "TheHive and its components are no longer part of Security Onion"

salt/common/tools/sbin/so-cortex-start

@@ -17,4 +17,4 @@
 
 . /usr/sbin/so-common
 
-/usr/sbin/so-start thehive $1
+echo "TheHive and its components are no longer part of Security Onion"

salt/common/tools/sbin/so-cortex-stop

@@ -17,4 +17,4 @@
 
 . /usr/sbin/so-common
 
-/usr/sbin/so-stop cortex $1
+echo "TheHive and its components are no longer part of Security Onion"

salt/common/tools/sbin/so-cortex-user-add

@@ -17,38 +17,4 @@
 
 . /usr/sbin/so-common
 
-usage() {
-    echo "Usage: $0 <new-user-name>"
-    echo ""
-    echo "Adds a new user to Cortex. The new password will be read from STDIN."
-    exit 1
-}
-
-if [ $# -ne 1 ]; then
-  usage
-fi
-
-USER=$1
-
-CORTEX_KEY=$(lookup_pillar cortexorguserkey)
-CORTEX_API_URL="$(lookup_pillar url_base)/cortex/api"
-CORTEX_ORG_NAME=$(lookup_pillar cortexorgname)
-CORTEX_USER=$USER
-
-# Read password for new user from stdin
-test -t 0
-if [[ $? == 0 ]]; then
-  echo "Enter new password:"
-fi
-read -rs CORTEX_PASS
-
-# Create new user in Cortex
-resp=$(curl -sk -XPOST -H "Authorization: Bearer $CORTEX_KEY" -H "Content-Type: application/json" -L "https://$CORTEX_API_URL/user" -d "{\"name\": \"$CORTEX_USER\",\"roles\": [\"read\",\"analyze\",\"orgadmin\"],\"organization\": \"$CORTEX_ORG_NAME\",\"login\": \"$CORTEX_USER\",\"password\" : \"$CORTEX_PASS\" }")
-if [[ "$resp" =~ \"status\":\"Ok\" ]]; then
-    echo "Successfully added user to Cortex."
-else
-    echo "Unable to add user to Cortex; user might already exist."
-    echo $resp
-    exit 2
-fi
-    
+echo "TheHive and its components are no longer part of Security Onion"

salt/common/tools/sbin/so-cortex-user-enable

@@ -17,41 +17,4 @@
 
 . /usr/sbin/so-common
 
-usage() {
-    echo "Usage: $0 <user-name> <true|false>"
-    echo ""
-    echo "Enables or disables a user in Cortex."
-    exit 1
-}
-
-if [ $# -ne 2 ]; then
-  usage
-fi
-
-USER=$1
-
-CORTEX_KEY=$(lookup_pillar cortexorguserkey)
-CORTEX_API_URL="$(lookup_pillar url_base)/cortex/api"
-CORTEX_USER=$USER
-
-case "${2^^}" in
-	FALSE | NO | 0)
-		CORTEX_STATUS=Locked
-		;;
-	TRUE | YES | 1)
-		CORTEX_STATUS=Ok
-		;;
-	*)
-		usage
-		;;
-esac
-
-resp=$(curl -sk -XPATCH -H "Authorization: Bearer $CORTEX_KEY" -H "Content-Type: application/json" -L "https://$CORTEX_API_URL/user/${CORTEX_USER}" -d "{\"status\":\"${CORTEX_STATUS}\" }")
-if [[ "$resp" =~ \"status\":\"Locked\" || "$resp" =~ \"status\":\"Ok\" ]]; then
-    echo "Successfully updated user in Cortex."
-else
-    echo "Failed to update user in Cortex."
-    echo $resp
-    exit 2
-fi
-    
+echo "TheHive and its components are no longer part of Security Onion"

salt/common/tools/sbin/so-elasticsearch-indices-rw

@@ -17,9 +17,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 IP={{ salt['grains.get']('ip_interfaces').get(salt['pillar.get']('sensor:mainint', salt['pillar.get']('manager:mainint', salt['pillar.get']('elasticsearch:mainint', salt['pillar.get']('host:mainint')))))[0] }}
 ESPORT=9200
-THEHIVEESPORT=9400
 
 echo "Removing read only attributes for indices..."
 echo
 {{ ELASTICCURL }} -s -k -XPUT -H "Content-Type: application/json" -L https://$IP:9200/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}' 2>&1 | if grep -q ack; then echo "Index settings updated..."; else echo "There was any issue updating the read-only attribute.  Please ensure Elasticsearch is running.";fi;
-{{ ELASTICCURL }} -XPUT -H "Content-Type: application/json" -L http://$IP:9400/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}' 2>&1 | if grep -q ack; then echo "Index settings updated..."; else echo "There was any issue updating the read-only attribute.  Please ensure Elasticsearch is running.";fi;

salt/common/tools/sbin/so-filebeat-module-setup

@@ -49,19 +49,18 @@ if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then
 fi
 echo "Testing to see if the pipelines are already applied"
 ESVER=$({{ ELASTICCURL }} -sk https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT" |jq .version.number |tr -d \")
-PIPELINES=$({{ ELASTICCURL }} -sk https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"/_ingest/pipeline/filebeat-$ESVER-suricata-eve-pipeline | jq . | wc -c)
+PIPELINES=$({{ ELASTICCURL }} -sk https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"/_ingest/pipeline/filebeat-$ESVER-elasticsearch-server-pipeline | jq . | wc -c)
 
-if [[ "$PIPELINES" -lt 5 ]]; then
+if [[ "$PIPELINES" -lt 5 ]] || [ "$2" != "--force" ]; then
   echo "Setting up ingest pipeline(s)"
-
-  for MODULE in activemq apache auditd aws azure barracuda bluecoat cef checkpoint cisco coredns crowdstrike cyberark cylance elasticsearch envoyproxy f5 fortinet gcp google_workspace googlecloud gsuite haproxy ibmmq icinga iis imperva infoblox iptables juniper kafka kibana logstash microsoft mongodb mssql mysql nats netscout nginx o365 okta osquery panw postgresql rabbitmq radware redis santa snort snyk sonicwall sophos squid suricata system threatintel tomcat traefik zeek zscaler
-  do
-    echo "Loading $MODULE"
-    docker exec -i so-filebeat filebeat setup modules -pipelines -modules $MODULE -c $FB_MODULE_YML
-    sleep 2
-  done
+{% from 'filebeat/modules.map.jinja' import MODULESMERGED with context %}
+{%- for module in MODULESMERGED.modules.keys() %}
+  {%- for fileset in MODULESMERGED.modules[module] %}
+      echo "{{ module }}.{{ fileset}}"
+      docker exec -i so-filebeat filebeat setup --pipelines --modules {{ module }} -M "{{ module }}.{{ fileset }}.enabled=true" -c $FB_MODULE_YML
+      sleep 0.5
+  {% endfor %}
+{%- endfor %}
 else
   exit 0
 fi
-
-

salt/common/tools/sbin/so-firewall

@@ -16,6 +16,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
+import re
 import subprocess
 import sys
 import time
@@ -26,6 +27,7 @@ hostgroupsFilename = "/opt/so/saltstack/local/salt/firewall/hostgroups.local.yam
 portgroupsFilename = "/opt/so/saltstack/local/salt/firewall/portgroups.local.yaml"
 defaultPortgroupsFilename = "/opt/so/saltstack/default/salt/firewall/portgroups.yaml"
 supportedProtocols = ['tcp', 'udp']
+readonly = False
 
 def showUsage(options, args):
   print('Usage: {} [OPTIONS] <COMMAND> [ARGS...]'.format(sys.argv[0]))
@@ -70,10 +72,26 @@ def checkApplyOption(options):
     return apply(None, None)
 
 def loadYaml(filename):
+  global readonly
+
   file = open(filename, "r")
-  return yaml.safe_load(file.read())
+  content = file.read()
+
+  # Remove Jinja templating (for read-only operations)
+  if "{%" in content or "{{" in content:
+    content = content.replace("{{ ssh_port }}", "22")
+    pattern = r'.*({%|{{|}}|%}).*'
+    content = re.sub(pattern, "", content)
+    readonly = True
+
+  return yaml.safe_load(content)
 
 def writeYaml(filename, content):
+  global readonly
+
+  if readonly:
+    raise Exception("Cannot write yaml file that has been flagged as read-only")  
+
   file = open(filename, "w")
   return yaml.dump(content, file)
 

salt/common/tools/sbin/so-image-common

@@ -75,9 +75,6 @@ container_list() {
       "so-strelka-manager"
       "so-suricata"
       "so-telegraf"
-      "so-thehive"
-      "so-thehive-cortex"
-      "so-thehive-es"
       "so-wazuh"
       "so-zeek" 
     )

salt/common/tools/sbin/so-kibana-space-defaults

@@ -1,6 +1,7 @@
+#!/bin/bash
 . /usr/sbin/so-common
 {% set HIGHLANDER = salt['pillar.get']('global:highlander', False) %}
-wait_for_web_response "http://localhost:5601/app/kibana" "Elastic" 300 "{{ ELASTICCURL }}"
+wait_for_web_response "http://localhost:5601/api/spaces/space/default" "default" 300 "{{ ELASTICCURL }}"
 ## This hackery will be removed if using Elastic Auth ##
 
 # Let's snag a cookie from Kibana
@@ -12,6 +13,6 @@ echo "Setting up default Space:"
 {% if HIGHLANDER %}
 {{ ELASTICCURL }} -b "sid=$SESSIONCOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["enterpriseSearch"]} ' >> /opt/so/log/kibana/misc.log
 {% else %}
-{{ ELASTICCURL }} -b "sid=$SESSIONCOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["ml","enterpriseSearch","siem","logs","infrastructure","apm","uptime","monitoring","stackAlerts","actions","fleet"]} ' >> /opt/so/log/kibana/misc.log
+{{ ELASTICCURL }} -b "sid=$SESSIONCOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["ml","enterpriseSearch","siem","logs","infrastructure","apm","uptime","monitoring","stackAlerts","actions","fleet","fleetv2","securitySolutionCases"]} ' >> /opt/so/log/kibana/misc.log
 {% endif %}
 echo

salt/common/tools/sbin/so-pcap-export

@@ -20,7 +20,7 @@ if [ $# -lt 2 ]; then
   exit 1
 fi
 
-docker exec -it so-sensoroni scripts/stenoquery.sh "$1" -w /nsm/pcapout/$2.pcap
+docker exec -t so-sensoroni scripts/stenoquery.sh "$1" -w /nsm/pcapout/$2.pcap
 
 echo ""
 echo "If successful, the output was written to: /nsm/pcapout/$2.pcap"

salt/common/tools/sbin/so-playbook-sync

@@ -18,7 +18,7 @@
 . /usr/sbin/so-common
 
 # Check to see if we are already running
-IS_RUNNING=$(ps aux | pgrep -f "so-playbook-sync" | wc -l)
-[ "$IS_RUNNING" -gt 3 ] && echo "$(date) - Multiple Playbook Sync processes already running...exiting." && exit 0
+NUM_RUNNING=$(pgrep -cf "/bin/bash /usr/sbin/so-playbook-sync")
+[ "$NUM_RUNNING" -gt 1 ] && echo "$(date) - $NUM_RUNNING Playbook sync processes running...exiting." && exit 0
 
 docker exec so-soctopus python3 playbook_play-sync.py

salt/common/tools/sbin/so-restart

@@ -15,7 +15,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-# Usage: so-restart  filebeat | kibana | playbook | thehive
+# Usage: so-restart  filebeat | kibana | playbook 
 
 . /usr/sbin/so-common
 
@@ -31,7 +31,6 @@ if [ $# -ge 1 ]; then
         fi
 
         case $1 in
-                "cortex") docker stop so-thehive-cortex so-thehive && docker rm so-thehive-cortex so-thehive && salt-call state.apply hive queue=True;;
                 "steno") docker stop so-steno && docker rm so-steno && salt-call state.apply pcap queue=True;;
                 *)  docker stop so-$1 ; docker rm so-$1 ; salt-call state.apply $1 queue=True;;
         esac

salt/common/tools/sbin/so-saltstack-update

@@ -32,11 +32,17 @@ copy_new_files() {
   # Copy new files over to the salt dir
   cd /tmp/sogh/securityonion
   git checkout $BRANCH
+  VERSION=$(cat VERSION)
+  # We need to overwrite if there is a repo file
+  if [ -d /opt/so/repo ]; then
+    tar -czf /opt/so/repo/"$VERSION".tar.gz -C "$(pwd)/.." .
+  fi
   rsync -a salt $default_salt_dir/
   rsync -a pillar $default_salt_dir/
   chown -R socore:socore $default_salt_dir/salt
   chown -R socore:socore $default_salt_dir/pillar
   chmod 755 $default_salt_dir/pillar/firewall/addfirewall.sh
+
   rm -rf /tmp/sogh
 }
 

salt/common/tools/sbin/so-sensor-clean

@@ -115,8 +115,8 @@ clean() {
 }
 
 # Check to see if we are already running
-IS_RUNNING=$(ps aux | pgrep -f "so-sensor-clean" | wc -l)
-[ "$IS_RUNNING" -gt 3 ] && echo "$(date) - $IS_RUNNING sensor clean script processes running...exiting." >>$LOG && exit 0
+NUM_RUNNING=$(pgrep -cf "/bin/bash /usr/sbin/so-sensor-clean")
+[ "$NUM_RUNNING" -gt 1 ] && echo "$(date) - $NUM_RUNNING sensor clean script processes running...exiting." >>$LOG && exit 0
 
 if [ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ]; then
 	while [ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ]; do

salt/common/tools/sbin/so-start

@@ -15,7 +15,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-# Usage: so-start  all | filebeat | kibana | playbook | thehive
+# Usage: so-start  all | filebeat | kibana | playbook 
 
 . /usr/sbin/so-common
 

salt/common/tools/sbin/so-thehive-es-restart

@@ -17,5 +17,4 @@
 
 . /usr/sbin/so-common
 
-/usr/sbin/so-stop thehive-es $1
-/usr/sbin/so-start thehive $1
+echo "TheHive and its components are no longer part of Security Onion"

salt/common/tools/sbin/so-thehive-es-start

@@ -17,4 +17,4 @@
 
 . /usr/sbin/so-common
 
-/usr/sbin/so-start thehive $1
+echo "TheHive and its components are no longer part of Security Onion"

salt/common/tools/sbin/so-thehive-es-stop

@@ -17,4 +17,4 @@
 
 . /usr/sbin/so-common
 
-/usr/sbin/so-stop thehive-es $1
+echo "TheHive and its components are no longer part of Security Onion"

salt/common/tools/sbin/so-thehive-restart

@@ -17,4 +17,4 @@
 
 . /usr/sbin/so-common
 
-/usr/sbin/so-restart thehive $1
+echo "TheHive and its components are no longer part of Security Onion"

salt/common/tools/sbin/so-thehive-start

@@ -17,4 +17,4 @@
 
 . /usr/sbin/so-common
 
-/usr/sbin/so-start thehive $1
+echo "TheHive and its components are no longer part of Security Onion"

salt/common/tools/sbin/so-thehive-stop

@@ -17,4 +17,4 @@
 
 . /usr/sbin/so-common
 
-/usr/sbin/so-stop thehive $1
+echo "TheHive and its components are no longer part of Security Onion"

salt/common/tools/sbin/so-thehive-user-add

@@ -17,38 +17,4 @@
 
 . /usr/sbin/so-common
 
-usage() {
-    echo "Usage: $0 <new-user-name>"
-    echo ""
-    echo "Adds a new user to TheHive. The new password will be read from STDIN."
-    exit 1
-}
-
-if [ $# -ne 1 ]; then
-  usage
-fi
-
-USER=$1
-
-THEHIVE_KEY=$(lookup_pillar hivekey)
-THEHVIE_API_URL="$(lookup_pillar url_base)/thehive/api"
-THEHIVE_USER=$USER
-
-# Read password for new user from stdin
-test -t 0
-if [[ $? == 0 ]]; then
-  echo "Enter new password:"
-fi
-read -rs THEHIVE_PASS
-
-check_password_and_exit "$THEHIVE_PASS"
-
-# Create new user in TheHive
-resp=$(curl -sk -XPOST -H "Authorization: Bearer $THEHIVE_KEY" -H "Content-Type: application/json" -L "https://$THEHVIE_API_URL/user" -d "{\"login\" : \"$THEHIVE_USER\",\"name\" : \"$THEHIVE_USER\",\"roles\" : [\"read\",\"alert\",\"write\",\"admin\"],\"preferences\" : \"{}\",\"password\" : \"$THEHIVE_PASS\"}")
-if [[ "$resp" =~ \"status\":\"Ok\" ]]; then
-    echo "Successfully added user to TheHive"
-else
-    echo "Unable to add user to TheHive; user might already exist"
-    echo $resp
-    exit 2
-fi
+echo "TheHive and its components are no longer part of Security Onion"

salt/common/tools/sbin/so-thehive-user-enable

@@ -17,41 +17,4 @@
 
 . /usr/sbin/so-common
 
-usage() {
-    echo "Usage: $0 <user-name> <true|false>"
-    echo ""
-    echo "Enables or disables a user in TheHive."
-    exit 1
-}
-
-if [ $# -ne 2 ]; then
-  usage
-fi
-
-USER=$1
-
-THEHIVE_KEY=$(lookup_pillar hivekey)
-THEHVIE_API_URL="$(lookup_pillar url_base)/thehive/api"
-THEHIVE_USER=$USER
-
-case "${2^^}" in
-	FALSE | NO | 0)
-		THEHIVE_STATUS=Locked
-		;;
-	TRUE | YES | 1)
-		THEHIVE_STATUS=Ok
-		;;
-	*)
-		usage
-		;;
-esac
-
-resp=$(curl -sk -XPATCH -H "Authorization: Bearer $THEHIVE_KEY" -H "Content-Type: application/json" -L "https://$THEHVIE_API_URL/user/${THEHIVE_USER}" -d "{\"status\":\"${THEHIVE_STATUS}\" }")
-if [[ "$resp" =~ \"status\":\"Locked\" || "$resp" =~ \"status\":\"Ok\" ]]; then
-    echo "Successfully updated user in TheHive"
-else
-    echo "Failed to update user in TheHive"
-    echo "$resp"
-    exit 2
-fi
-    
+echo "TheHive and its components are no longer part of Security Onion"

salt/common/tools/sbin/so-thehive-user-update

@@ -17,41 +17,4 @@
 
 . /usr/sbin/so-common
 
-usage() {
-    echo "Usage: $0 <user-name>"
-    echo ""
-    echo "Update password for an existing TheHive user. The new password will be read from STDIN."
-    exit 1
-}
-
-if [ $# -ne 1 ]; then
-  usage
-fi
-
-USER=$1
-
-THEHIVE_KEY=$(lookup_pillar hivekey)
-THEHVIE_API_URL="$(lookup_pillar url_base)/thehive/api"
-THEHIVE_USER=$USER
-
-# Read password for new user from stdin
-test -t 0
-if [[ $? == 0 ]]; then
-  echo "Enter new password:"
-fi
-read -rs THEHIVE_PASS
-
-if ! check_password "$THEHIVE_PASS"; then
-  echo "Password is invalid. Please exclude single quotes, double quotes, dollar signs, and backslashes from the password."
-  exit 2
-fi
-
-# Change password for user in TheHive
-resp=$(curl -sk -XPOST -H "Authorization: Bearer $THEHIVE_KEY" -H "Content-Type: application/json" -L "https://$THEHVIE_API_URL/user/${THEHIVE_USER}/password/set" -d "{\"password\" : \"$THEHIVE_PASS\"}")
-if [[ -z "$resp" ]]; then
-    echo "Successfully updated TheHive user password"
-else
-    echo "Unable to update TheHive user password"
-    echo $resp
-    exit 2
-fi
+echo "TheHive and its components are no longer part of Security Onion"

salt/common/tools/sbin/so-user

@@ -44,7 +44,7 @@ operation=$1
 email=$2
 role=$3
 
-kratosUrl=${KRATOS_URL:-http://127.0.0.1:4434}
+kratosUrl=${KRATOS_URL:-http://127.0.0.1:4434/admin}
 databasePath=${KRATOS_DB_PATH:-/opt/so/conf/kratos/db/db.sqlite}
 databaseTimeout=${KRATOS_DB_TIMEOUT:-5000}
 bcryptRounds=${BCRYPT_ROUNDS:-12}
@@ -238,7 +238,7 @@ function syncElastic() {
   syncElasticSystemUser "$authPillarJson" "so_monitor_user"  "$usersTmpFile"
 
   syncElasticSystemRole "$authPillarJson" "so_elastic_user"  "superuser" "$rolesTmpFile"
-  syncElasticSystemRole "$authPillarJson" "so_kibana_user"   "superuser" "$rolesTmpFile"
+  syncElasticSystemRole "$authPillarJson" "so_kibana_user"   "kibana_system" "$rolesTmpFile"
   syncElasticSystemRole "$authPillarJson" "so_logstash_user" "superuser" "$rolesTmpFile"
   syncElasticSystemRole "$authPillarJson" "so_beats_user"    "superuser" "$rolesTmpFile"
   syncElasticSystemRole "$authPillarJson" "so_monitor_user"  "remote_monitoring_collector" "$rolesTmpFile"
@@ -408,7 +408,7 @@ function migrateLockedUsers() {
   # This is a migration function to convert locked users from prior to 2.3.90
   # to inactive users using the newer Kratos functionality. This should only
   # find locked users once.
-  lockedEmails=$(curl -s http://localhost:4434/identities | jq -r '.[] | select(.traits.status == "locked") | .traits.email')
+  lockedEmails=$(curl -s ${kratosUrl}/identities | jq -r '.[] | select(.traits.status == "locked") | .traits.email')
   if [[ -n "$lockedEmails" ]]; then
     echo "Disabling locked users..."
     for email in $lockedEmails; do
@@ -437,7 +437,7 @@ function updateStatus() {
     state="inactive"
   fi
   body="{ \"schema_id\": \"$schemaId\", \"state\": \"$state\", \"traits\": $traitBlock }"
-  response=$(curl -fSsL -XPUT "${kratosUrl}/identities/$identityId" -d "$body")
+  response=$(curl -fSsL -XPUT -H "Content-Type: application/json" "${kratosUrl}/identities/$identityId" -d "$body")
   [[ $? != 0 ]] && fail "Unable to update user"
 }
 
@@ -476,7 +476,6 @@ case "${operation}" in
     createUser "$email" "${role:-$DEFAULT_ROLE}"
     syncAll
     echo "Successfully added new user to SOC"
-    check_container thehive && echo "$password" | so-thehive-user-add "$email"
     check_container fleet && echo "$password" | so-fleet-user-add "$email"
     ;;
 
@@ -528,7 +527,6 @@ case "${operation}" in
     updateStatus "$email" 'active'
     syncAll
     echo "Successfully enabled user"
-    check_container thehive && so-thehive-user-enable "$email" true
     echo "Fleet user will need to be recreated manually with so-fleet-user-add"
     ;;
 
@@ -540,7 +538,6 @@ case "${operation}" in
     updateStatus "$email" 'locked'
     syncAll
     echo "Successfully disabled user"
-    check_container thehive && so-thehive-user-enable "$email" false
     check_container fleet && so-fleet-user-delete "$email"   
     ;;    
 
@@ -552,7 +549,6 @@ case "${operation}" in
     deleteUser "$email"
     syncAll
     echo "Successfully deleted user"
-    check_container thehive && so-thehive-user-enable "$email" false
     check_container fleet && so-fleet-user-delete "$email"
     ;;
 

salt/common/tools/sbin/so-yara-update

@@ -48,7 +48,7 @@ fi
 
 {% else %}
 
-gh_status=$(curl -s -o /dev/null -w "%{http_code}" http://github.com)
+gh_status=$(curl -s -o /dev/null -w "%{http_code}" https://github.com)
 clone_dir="/tmp"
 if [ "$gh_status" == "200" ]  || [ "$gh_status" == "301" ]; then
 

salt/common/tools/sbin/so-zeek-logs

@@ -10,39 +10,118 @@ zeek_logs_enabled() {
 }
 
 whiptail_manager_adv_service_zeeklogs() {
-  BLOGS=$(whiptail --title "so-zeek-logs" --checklist "Please Select Logs to Send:" 24 78 12 \
-  "conn" "Connection Logging" ON \
-  "dce_rpc" "RPC Logs" ON \
-  "dhcp" "DHCP Logs" ON \
-  "dnp3" "DNP3 Logs" ON \
-  "dns" "DNS Logs" ON \
-  "dpd" "DPD Logs" ON \
-  "files" "Files Logs" ON \
-  "ftp" "FTP Logs" ON \
-  "http" "HTTP Logs" ON \
-  "intel" "Intel Hits Logs" ON \
-  "irc" "IRC Chat Logs" ON \
-  "kerberos" "Kerberos Logs" ON \
-  "modbus" "MODBUS Logs" ON \
-  "notice" "Zeek Notice Logs" ON \
-  "ntlm" "NTLM Logs" ON \
-  "pe" "PE Logs" ON \
-  "radius" "Radius Logs" ON \
-  "rfb" "RFB Logs" ON \
-  "rdp" "RDP Logs" ON \
-  "sip" "SIP Logs" ON \
-  "smb_files" "SMB Files Logs" ON \
-  "smb_mapping" "SMB Mapping Logs" ON \
-  "smtp" "SMTP Logs" ON \
-  "snmp" "SNMP Logs" ON \
-  "ssh" "SSH Logs" ON \
-  "ssl" "SSL Logs" ON \
-  "syslog" "Syslog Logs" ON \
-  "tunnel" "Tunnel Logs" ON \
-  "weird" "Zeek Weird Logs" ON \
-  "mysql" "MySQL Logs" ON \
-  "socks" "SOCKS Logs" ON \
-  "x509" "x.509 Logs" ON 3>&1 1>&2 2>&3 )
+  BLOGS=$(whiptail --title "so-zeek-logs" --checklist "Please select logs to send:" 24 78 12 \
+  "conn" "" ON \
+  "dce_rpc" "" ON \
+  "dhcp" "" ON \
+  "dnp3" "" ON \
+  "dns" "" ON \
+  "dpd" "" ON \
+  "files" "" ON \
+  "ftp" "" ON \
+  "http" "" ON \
+  "intel" "" ON \
+  "irc" "" ON \
+  "kerberos" "" ON \
+  "modbus" "" ON \
+  "notice" "" ON \
+  "ntlm" "" ON \
+  "pe" "" ON \
+  "radius" "" ON \
+  "rfb" "" ON \
+  "rdp" "" ON \
+  "sip" "" ON \
+  "smb_files" "" ON \
+  "smb_mapping" "" ON \
+  "smtp" "" ON \
+  "snmp" "" ON \
+  "software" "" ON \
+  "ssh" "" ON \
+  "ssl" "" ON \
+  "syslog" "" ON \
+  "tunnel" "" ON \
+  "weird" "" ON \
+  "mysql" "" ON \
+  "socks" "" ON \
+  "x509" "" ON \
+  "bacnet" "" ON \
+  "bacnet_discovery" "" ON \
+  "bacnet_property" "" ON \
+  "bsap_ip_header" "" ON \
+  "bsap_ip_rdb" "" ON \
+  "bsap_ip_unknown" "" ON \
+  "bsap_serial_header" "" ON \
+  "bsap_serial_rdb" "" ON \
+  "bsap_serial_rdb_ext" "" ON \
+  "bsap_serial_unknown" "" ON \
+  "cip" "" ON \
+  "cip_identity" "" ON \
+  "cip_io" "" ON \
+  "cotp" "" ON \
+  "dnp3_control" "" ON \
+  "dnp3_objects" "" ON \
+  "ecat_aoe_info" "" ON \
+  "ecat_arp_info" "" OFF \
+  "ecat_coe_info" "" ON \
+  "ecat_dev_info" "" ON \
+  "ecat_foe_info" "" ON \
+  "ecat_log_address" "" ON \
+  "ecat_registers" "" ON \
+  "ecat_soe_info" "" ON \
+  "enip" "" ON \
+  "modbus_detailed" "" ON \
+  "modbus_mask_write_register" "" ON \
+  "modbus_read_write_multiple_registers" "" ON \
+  "opcua_binary" "" ON \
+  "opcua_binary_activate_session" "" ON \
+  "opcua_binary_activate_session_client_software_cert" "" ON \
+  "opcua_binary_activate_session_diagnostic_info" "" ON \
+  "opcua_binary_activate_session_locale_id" "" ON \
+  "opcua_binary_browse" "" ON \
+  "opcua_binary_browse_description" "" ON \
+  "opcua_binary_browse_diagnostic_info" "" ON \
+  "opcua_binary_browse_request_continuation_point" "" ON \
+  "opcua_binary_browse_response_references" "" ON \
+  "opcua_binary_browse_result" "" ON \
+  "opcua_binary_create_session" "" ON \
+  "opcua_binary_create_session_discovery" "" ON \
+  "opcua_binary_create_session_endpoints" "" ON \
+  "opcua_binary_create_session_user_token" "" ON \
+  "opcua_binary_create_subscription" "" ON \
+  "opcua_binary_diag_info_detail" "" ON \
+  "opcua_binary_get_endpoints" "" ON \
+  "opcua_binary_get_endpoints_description" "" ON \
+  "opcua_binary_get_endpoints_discovery" "" ON \
+  "opcua_binary_get_endpoints_locale_id" "" ON \
+  "opcua_binary_get_endpoints_profile_uri" "" ON \
+  "opcua_binary_get_endpoints_user_token" "" ON \
+  "opcua_binary_opensecure_channel" "" ON \
+  "opcua_binary_read" "" ON \
+  "opcua_binary_read_array_dims" "" ON \
+  "opcua_binary_read_array_dims_link" "" ON \
+  "opcua_binary_read_diagnostic_info" "" ON \
+  "opcua_binary_read_extension_object" "" ON \
+  "opcua_binary_read_extension_object_link" "" ON \
+  "opcua_binary_read_nodes_to_read" "" ON \
+  "opcua_binary_read_results" "" ON \
+  "opcua_binary_read_results_link" "" ON \
+  "opcua_binary_read_status_code" "" ON \
+  "opcua_binary_read_variant_data" "" ON \
+  "opcua_binary_read_variant_data_link" "" ON \
+  "opcua_binary_status_code_detail" "" ON \
+  "profinet" "" ON \
+  "profinet_dce_rpc" "" ON \
+  "profinet_debug" "" ON \
+  "s7comm" "" ON \
+  "s7comm_plus" "" ON \
+  "s7comm_read_szl" "" ON \
+  "s7comm_upload_download" "" ON \
+  "stun" "" ON \
+  "stun_nat" "" ON \
+  "tds" "" ON \
+  "tds_rpc" "" ON \
+  "tds_sql_batch" "" ON \
+  "wireguard" "" ON 3>&1 1>&2 2>&3 )
   
   local exitstatus=$?
 

salt/common/tools/sbin/soup

@@ -34,7 +34,15 @@ check_err() {
   local err_msg="Unhandled error occured, please check $SOUP_LOG for details."
 
   [[ $ERR_HANDLED == true ]] && exit $exit_code
+
   if [[ $exit_code -ne 0 ]]; then
+  
+    set +e
+    systemctl_func "start" "$cron_service_name"
+    systemctl_func "start" "salt-master"
+    systemctl_func "start" "salt-minion"
+    enable_highstate
+
     printf '%s' "Soup failed with error $exit_code: "
     case $exit_code in
       2)
@@ -91,9 +99,7 @@ check_err() {
     if [[ $exit_code -ge 64 && $exit_code -le 113 ]]; then
       echo "$err_msg"
     fi
-    set +e
-    systemctl_func "start" "$cron_service_name"
-    enable_highstate
+
     exit $exit_code
   fi
 
@@ -197,7 +203,7 @@ check_airgap() {
 
 check_local_mods() {
   local salt_local=/opt/so/saltstack/local
-
+  local_ignore_arr=("/opt/so/saltstack/local/salt/zeek/policy/intel/intel.dat")
   local_mod_arr=()
 
   while IFS= read -r -d '' local_file; do
@@ -205,8 +211,10 @@ check_local_mods() {
     default_file="${DEFAULT_SALT_DIR}${stripped_path}"
     if [[ -f $default_file ]]; then
       file_diff=$(diff "$default_file" "$local_file" )
-      if [[ $(echo "$file_diff" | grep -c "^<") -gt 0 ]]; then
-        local_mod_arr+=( "$local_file" )
+      if [[ ! " ${local_ignore_arr[*]} " =~ " ${local_file} " ]]; then
+        if [[ $(echo "$file_diff" | grep -c "^<") -gt 0 ]]; then
+          local_mod_arr+=( "$local_file" )
+        fi
       fi
     fi
   done< <(find $salt_local -type f -print0)
@@ -217,11 +225,24 @@ check_local_mods() {
       echo "  $file_str"
     done
     echo ""
-    echo "To reference this list later, check $SOUP_LOG"
-    sleep 10
+    echo "To reference this list later, check $SOUP_LOG".
+    echo
+    if [[ -z $UNATTENDED ]] && ! [[ "${1}" == "skip-prompt" ]]; then
+      while true; do
+        read -p "Please review the local modifications shown above as they may cause problems during or after the update.
+
+Would you like to proceed with the update anyway?
+
+If so, type 'YES'. Otherwise, type anything else to exit SOUP. " yn
+
+        case $yn in
+          [yY][eE][sS] ) echo "Local modifications accepted.  Continuing..."; break;;
+          * ) exit 0;;
+        esac
+      done
+    fi
   fi
 }
-
 # {% endraw %}
 
 check_pillar_items() {
@@ -365,12 +386,116 @@ clone_to_tmp() {
   fi
 }
 
+elastalert_indices_check() {
+  echo "Checking Elastalert indices for compatibility..."
+  # Wait for ElasticSearch to initialize
+  echo -n "Waiting for ElasticSearch..."
+  COUNT=0
+  ELASTICSEARCH_CONNECTED="no"
+  while [[ "$COUNT" -le 240 ]]; do
+    so-elasticsearch-query / -k --output /dev/null
+      if [ $? -eq 0 ]; then
+        ELASTICSEARCH_CONNECTED="yes"
+        echo "connected!"
+          break
+      else
+        ((COUNT+=1))
+        sleep 1
+        echo -n "."
+      fi
+  done
+
+  # Unable to connect to Elasticsearch
+  if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then
+    echo
+    echo -e "Connection attempt timed out.  Unable to connect to ElasticSearch.  \nPlease try: \n  -checking log(s) in /var/log/elasticsearch/\n  -running 'sudo docker ps' \n  -running 'sudo so-elastic-restart'"
+    echo
+    exit 1
+  fi
+
+  MAJOR_ES_VERSION=$(so-elasticsearch-query / | jq -r .version.number | cut -d '.' -f1)
+  if [[ "$MAJOR_ES_VERSION" -lt "8" ]]; then
+
+    # Stop Elastalert to prevent Elastalert indices from being re-created
+    if grep -q "^so-elastalert$" /opt/so/conf/so-status/so-status.conf ; then
+      so-elastalert-stop || true
+    fi
+
+    # Check Elastalert indices
+    echo "Deleting Elastalert indices to prevent issues with upgrade to Elastic 8..."
+    CHECK_COUNT=0
+    while [[ "$CHECK_COUNT" -le 2 ]]; do
+      # Delete Elastalert indices
+      for i in $(so-elasticsearch-query _cat/indices | grep elastalert | awk '{print $3}'); do
+        so-elasticsearch-query $i -XDELETE;
+         done
+
+      # Check to ensure Elastalert indices are deleted
+      COUNT=0
+      ELASTALERT_INDICES_DELETED="no"
+      while [[ "$COUNT" -le 240 ]]; do
+        RESPONSE=$(so-elasticsearch-query "elastalert*")
+        if [[ "$RESPONSE" == "{}" ]]; then
+          ELASTALERT_INDICES_DELETED="yes"
+          break
+        else
+          ((COUNT+=1))
+          sleep 1
+          echo -n "."
+        fi
+      done
+      ((CHECK_COUNT+=1))
+    done
+
+    # If we were unable to delete the Elastalert indices, exit the script
+    if [ "$ELASTALERT_INDICES_DELETED" == "yes" ]; then
+      echo "Elastalert indices successfully deleted."
+    else
+      echo
+      echo -e "Unable to connect to delete Elastalert indices. Exiting."
+      echo
+      exit 1
+    fi
+  else
+    echo "Major Elasticsearch version is 8 or greater...skipping Elastalert index maintenance."
+  fi
+}
+
 enable_highstate() {
     echo "Enabling highstate."
     salt-call state.enable highstate -l info --local
     echo ""
 }
 
+es_version_check() {
+    CHECK_ES=$(echo $INSTALLEDVERSION | awk -F. '{print $3}')
+  
+    if [[ "$CHECK_ES" -lt "110" ]]; then
+      echo "You are currently running Security Onion $INSTALLEDVERSION. You will need to update to version 2.3.130 before updating to 2.3.140 or higher."
+      echo ""
+      echo "If your deployment has Internet access, you can use the following command to update to 2.3.130:"
+      echo "sudo BRANCH=2.3.130-20220607 soup"
+      echo ""
+      echo "Otherwise, if your deployment is configured for airgap, you can instead download the 2.3.130 ISO image from https://download.securityonion.net/file/securityonion/securityonion-2.3.130-20220607.iso."
+      echo ""
+      echo "*** Once you have updated to 2.3.130, you can then update to 2.3.140 or higher as you would normally. ***"
+      exit 1
+    fi
+}
+
+es_indices_check() {
+  echo "Checking for unsupported Elasticsearch indices..."
+  UNSUPPORTED_INDICES=$(for INDEX in $(so-elasticsearch-indices-list | awk '{print $3}'); do so-elasticsearch-query $INDEX/_settings?human |grep '"created_string":"6' | jq -r 'keys'[0]; done)
+  if [ -z "$UNSUPPORTED_INDICES" ]; then
+    echo "No unsupported indices found."
+  else
+    echo "The following indices were created with Elasticsearch 6, and are not supported when upgrading to Elasticsearch 8. These indices may need to be deleted, migrated, or re-indexed before proceeding with the upgrade. Please see https://docs.securityonion.net/en/2.3/soup.html#elastic-8 for more details."
+    echo
+    echo "$UNSUPPORTED_INDICES"
+    exit 1
+  fi
+}
+
 generate_and_clean_tarballs() {
   local new_version
   new_version=$(cat $UPDATE_DIR/VERSION)
@@ -416,6 +541,16 @@ preupgrade_changes() {
     [[ "$INSTALLEDVERSION" == 2.3.80 ]] && up_to_2.3.90
     [[ "$INSTALLEDVERSION" == 2.3.90 || "$INSTALLEDVERSION" == 2.3.91 ]] && up_to_2.3.100
     [[ "$INSTALLEDVERSION" == 2.3.100 ]] && up_to_2.3.110
+    [[ "$INSTALLEDVERSION" == 2.3.110 ]] && up_to_2.3.120
+    [[ "$INSTALLEDVERSION" == 2.3.120 ]] && up_to_2.3.130
+    [[ "$INSTALLEDVERSION" == 2.3.130 ]] && up_to_2.3.140
+    [[ "$INSTALLEDVERSION" == 2.3.140 ]] && up_to_2.3.150
+    [[ "$INSTALLEDVERSION" == 2.3.150 ]] && up_to_2.3.160
+    [[ "$INSTALLEDVERSION" == 2.3.160 ]] && up_to_2.3.170
+    [[ "$INSTALLEDVERSION" == 2.3.170 ]] && up_to_2.3.180
+    [[ "$INSTALLEDVERSION" == 2.3.180 ]] && up_to_2.3.181
+    [[ "$INSTALLEDVERSION" == 2.3.181 ]] && up_to_2.3.182
+    [[ "$INSTALLEDVERSION" == 2.3.182 ]] && up_to_2.3.190
     true
 }
 
@@ -429,6 +564,17 @@ postupgrade_changes() {
     [[ "$POSTVERSION" == 2.3.60 || "$POSTVERSION" == 2.3.61 || "$POSTVERSION" == 2.3.70 || "$POSTVERSION" == 2.3.80 ]] && post_to_2.3.90
     [[ "$POSTVERSION" == 2.3.90 || "$POSTVERSION" == 2.3.91 ]] && post_to_2.3.100
     [[ "$POSTVERSION" == 2.3.100 ]] && post_to_2.3.110
+    [[ "$POSTVERSION" == 2.3.110 ]] && post_to_2.3.120
+    [[ "$POSTVERSION" == 2.3.120 ]] && post_to_2.3.130
+    [[ "$POSTVERSION" == 2.3.130 ]] && post_to_2.3.140
+    [[ "$POSTVERSION" == 2.3.140 ]] && post_to_2.3.150
+    [[ "$POSTVERSION" == 2.3.150 ]] && post_to_2.3.160
+    [[ "$POSTVERSION" == 2.3.160 ]] && post_to_2.3.170
+    [[ "$POSTVERSION" == 2.3.170 ]] && post_to_2.3.180
+    [[ "$POSTVERSION" == 2.3.180 ]] && post_to_2.3.181
+    [[ "$POSTVERSION" == 2.3.181 ]] && post_to_2.3.182
+    [[ "$POSTVERSION" == 2.3.182 ]] && post_to_2.3.190
+
     true
 }
 
@@ -492,6 +638,60 @@ post_to_2.3.110() {
   POSTVERSION=2.3.110
 }
 
+post_to_2.3.120() {
+  echo "Post Processing for 2.3.120"
+  POSTVERSION=2.3.120
+  sed -i '/so-thehive-es/d;/so-thehive/d;/so-cortex/d' /opt/so/conf/so-status/so-status.conf
+}
+
+post_to_2.3.130() {
+  echo "Post Processing for 2.3.130"
+  POSTVERSION=2.3.130
+}
+
+post_to_2.3.140() {
+  echo "Post Processing for 2.3.140"
+  FORCE_SYNC=true so-user sync
+  so-kibana-restart
+  so-kibana-space-defaults
+  POSTVERSION=2.3.140
+}
+
+post_to_2.3.150() {
+  echo "Nothing to do for .150"
+  POSTVERSION=2.3.150
+}
+
+post_to_2.3.160() {
+  echo "Nothing to do for .160"
+  POSTVERSION=2.3.160
+}
+
+post_to_2.3.170() {
+  echo "Nothing to do for .170"
+  POSTVERSION=2.3.170
+}
+
+post_to_2.3.180() {
+  echo "Nothing to do for .180"
+  POSTVERSION=2.3.180
+}
+
+post_to_2.3.181() {
+  echo "Nothing to do for .181"
+  POSTVERSION=2.3.181
+}
+
+post_to_2.3.182() {
+  echo "Nothing to do for .182"
+  POSTVERSION=2.3.182
+}
+
+post_to_2.3.190() {
+  echo "Nothing to do for .190"
+  POSTVERSION=2.3.190
+}
+
 stop_salt_master() {
     # kill all salt jobs across the grid because the hang indefinitely if they are queued and salt-master restarts
     set +e
@@ -728,9 +928,6 @@ up_to_2.3.90() {
 up_to_2.3.100() {
   fix_wazuh
   
-  echo "Removing /opt/so/state files for patched Salt InfluxDB module and state. This is due to Salt being upgraded and needing to patch the files again."
-  rm -vrf /opt/so/state/influxdb_continuous_query.py.patched /opt/so/state/influxdb_retention_policy.py.patched /opt/so/state/influxdbmod.py.patched
-  
   echo "Adding receiver hostgroup with so-firewall"
   if so-firewall addhostgroup receiver 2>&1 | grep -q 'Already exists'; then
     echo 'receiver hostgroup already exists'
@@ -740,12 +937,72 @@ up_to_2.3.100() {
 
   echo "Adding receiver to assigned_hostgroups.local.map.yaml"
   grep -qxF "  receiver:" /opt/so/saltstack/local/salt/firewall/assigned_hostgroups.local.map.yaml || sed -i -e '$a\  receiver:' /opt/so/saltstack/local/salt/firewall/assigned_hostgroups.local.map.yaml
+  
+  INSTALLEDVERSION=2.3.100
 }
 
 up_to_2.3.110() {
-  echo "Updating to Security Onion 2.3.110"
-  echo "Updating shard settings for Elasticsearch index templates"
   sed -i 's|shards|index_template:\n        template:\n          settings:\n            index:\n              number_of_shards|g' /opt/so/saltstack/local/pillar/global.sls
+  INSTALLEDVERSION=2.3.110
+}
+
+up_to_2.3.120() {
+  # Stop thehive services since these will be broken in .120
+  so-thehive-stop
+  so-thehive-es-stop
+  so-cortex-stop
+  INSTALLEDVERSION=2.3.120
+}
+
+up_to_2.3.130() {
+  # Remove file for nav update
+  rm -f /opt/so/conf/navigator/layers/nav_layer_playbook.json
+  INSTALLEDVERSION=2.3.130
+}
+
+up_to_2.3.140() {
+   elastalert_indices_check
+   ##
+   INSTALLEDVERSION=2.3.140
+}
+
+up_to_2.3.150() {
+  echo "Upgrading to 2.3.150"
+  INSTALLEDVERSION=2.3.150
+}
+
+up_to_2.3.160() {
+  echo "Upgrading to 2.3.160"
+  INSTALLEDVERSION=2.3.160
+}
+
+up_to_2.3.170() {
+  echo "Upgrading to 2.3.170"
+  INSTALLEDVERSION=2.3.170
+}
+
+up_to_2.3.180() {
+  echo "Upgrading to 2.3.180"
+  INSTALLEDVERSION=2.3.180
+}
+
+up_to_2.3.181() {
+  echo "Upgrading to 2.3.181"
+  INSTALLEDVERSION=2.3.181
+}
+
+up_to_2.3.182() {
+  echo "Upgrading to 2.3.182"
+  INSTALLEDVERSION=2.3.182
+}
+
+up_to_2.3.190() {
+  echo "Upgrading to 2.3.190"
+  if [ -d /nsm/zeek/extracted/complete ]; then
+    chown -R zeek:socore /nsm/zeek/extracted/complete
+    chmod 770 /nsm/zeek/extracted/complete
+  fi
+  INSTALLEDVERSION=2.3.190
 }
 
 verify_upgradespace() {
@@ -770,29 +1027,6 @@ upgrade_space() {
   fi  
 }
 
-thehive_maint() {
-  echo -n "Waiting for TheHive..."
-  COUNT=0
-  THEHIVE_CONNECTED="no"
-  while [[ "$COUNT" -le 240 ]]; do
-    curl --output /dev/null --silent --head --fail -k "https://localhost/thehive/api/alert"
-      if [ $? -eq 0 ]; then
-        THEHIVE_CONNECTED="yes"
-        echo "connected!"
-        break
-      else
-        ((COUNT+=1))
-        sleep 1
-        echo -n "."
-      fi
-  done
-  if [ "$THEHIVE_CONNECTED" == "yes" ]; then
-    echo "Migrating thehive databases if needed."
-    curl -v -k -XPOST -L "https://localhost/thehive/api/maintenance/migrate" >> "$SOUP_LOG" 2>&1
-    curl -v -k -XPOST -L "https://localhost/cortex/api/maintenance/migrate" >> "$SOUP_LOG" 2>&1
-  fi
-}
-
 unmount_update() {
   cd /tmp
   umount /tmp/soagupdate
@@ -908,6 +1142,8 @@ upgrade_salt() {
   else
     echo "Salt upgrade success."
     echo ""
+    echo "Removing /opt/so/state files for patched Salt InfluxDB module and state. This is due to Salt being upgraded and needing to patch the files again."
+    rm -vrf /opt/so/state/influxdb_continuous_query.py.patched /opt/so/state/influxdb_retention_policy.py.patched /opt/so/state/influxdbmod.py.patched
   fi
 
 }
@@ -947,7 +1183,7 @@ update_repo() {
     fi
 
     rm -f /etc/apt/sources.list.d/salt.list
-    echo "deb https://repo.securityonion.net/file/securityonion-repo/ubuntu/$ubuntu_version/amd64/salt $OSVER main" > /etc/apt/sources.list.d/saltstack.list
+    echo "deb https://repo.securityonion.net/file/securityonion-repo/ubuntu/$ubuntu_version/amd64/salt3004.2/ $OSVER main" > /etc/apt/sources.list.d/saltstack.list
     apt-get update
   fi
 }
@@ -1037,6 +1273,17 @@ main() {
   echo "### Preparing soup at $(date) ###"
   echo ""
 
+  set_os
+  set_cron_service_name
+  if ! check_salt_master_status; then
+  	echo "Could not talk to salt master"
+    echo "Please run 'systemctl status salt-master' to ensure the salt-master service is running and check the log at /opt/so/log/salt/master."
+    echo "SOUP will now attempt to start the salt-master service and exit."
+    exit 1
+  fi
+
+  echo "This node can communicate with the salt-master."
+
   echo "Checking to see if this is a manager."
   echo ""
   require_manager
@@ -1071,12 +1318,14 @@ main() {
   fi
   echo "Verifying we have the latest soup script."
   verify_latest_update_script
+  es_version_check
+  es_indices_check
+  elastalert_indices_check
   echo ""
-  set_os
-  set_cron_service_name
   set_palette
   check_elastic_license
   echo ""
+  check_local_mods
   check_os_updates
 
   echo "Generating new repo archive"
@@ -1222,7 +1471,6 @@ main() {
     salt-call state.highstate -l info queue=True
     postupgrade_changes
     [[ $is_airgap -eq 0 ]] && unmount_update
-    thehive_maint
 
     echo ""
     echo "Upgrade to $NEWVERSION complete."
@@ -1242,7 +1490,7 @@ main() {
     fi
 
     echo "Checking for local modifications."
-    check_local_mods
+    check_local_mods skip-prompt
 
     echo "Checking sudoers file."
     check_sudoers

salt/curator/files/action/so-kratos-close.yml

@@ -0,0 +1,29 @@
+{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-kratos:close', 30) -%}
+---
+# Remember, leave a key empty if there is no value.  None will be a string,
+# not a Python "NoneType"
+#
+# Also remember that all examples have 'disable_action' set to True.  If you
+# want to use this action as a template, be sure to set this to False after
+# copying it.
+actions:
+  1:
+    action: close
+    description: >-
+      Close kratos indices older than {{cur_close_days}} days.
+    options:
+      delete_aliases: False
+      timeout_override:
+      continue_if_exception: False
+      disable_action: False
+    filters:
+    - filtertype: pattern
+      kind: regex
+      value: '^(logstash-kratos.*|so-kratos.*)$'
+    - filtertype: age
+      source: name
+      direction: older
+      timestring: '%Y.%m.%d'
+      unit: days
+      unit_count: {{cur_close_days}}
+      exclude:

salt/curator/files/action/so-kratos-delete.yml

@@ -0,0 +1,29 @@
+{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-kratos:delete', 365) -%}
+---
+# Remember, leave a key empty if there is no value.  None will be a string,
+# not a Python "NoneType"
+#
+# Also remember that all examples have 'disable_action' set to True.  If you
+# want to use this action as a template, be sure to set this to False after
+# copying it.
+actions:
+  1:
+    action: delete_indices
+    description: >-
+      Delete kratos indices when older than {{ DELETE_DAYS }} days.
+    options:
+      ignore_empty_list: True
+      disable_action: False
+    filters:
+    - filtertype: pattern
+      kind: regex
+      value: '^(logstash-kratos.*|so-kratos.*)$'
+    - filtertype: age
+      source: name
+      direction: older
+      timestring: '%Y.%m.%d'
+      unit: days
+      unit_count: {{ DELETE_DAYS }}
+      exclude:
+  
+      

salt/curator/files/action/so-kratos-warm.yml

@@ -0,0 +1,24 @@
+{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-kratos:warm', 7) -%}
+actions:
+  1:
+    action: allocation
+    description: "Apply shard allocation filtering rules to the specified indices"
+    options:
+      key: box_type
+      value: warm
+      allocation_type: require
+      wait_for_completion: true
+      timeout_override:
+      continue_if_exception: false
+      disable_action: false
+    filters:
+    - filtertype: pattern
+      kind: prefix
+      value: so-kratos
+    - filtertype: age
+      source: name
+      direction: older
+      timestring: '%Y.%m.%d'
+      unit: days
+      unit_count: {{ WARM_DAYS }} 
+

salt/curator/files/bin/so-curator-close

@@ -23,22 +23,21 @@ read lastPID < $lf
 # if lastPID is not null and a process with that pid exists , exit
 [ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit
 echo $$ > $lf
-{% from 'filebeat/map.jinja' import THIRDPARTY with context %}
-{% from 'filebeat/map.jinja' import SO with context %}
+
+{% from 'filebeat/modules.map.jinja' import MODULESMERGED with context %}
 
 /usr/sbin/so-curator-closed-delete > /dev/null 2>&1; 
 docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-zeek-close.yml > /dev/null 2>&1; 
 docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-beats-close.yml > /dev/null 2>&1; 
 docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-firewall-close.yml > /dev/null 2>&1; 
 docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-ids-close.yml > /dev/null 2>&1; 
-docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-import-close.yml > /dev/null 2>&1; 
+docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-import-close.yml > /dev/null 2>&1;
+docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-kibana-close.yml > /dev/null 2>&1;
+docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-kratos-close.yml > /dev/null 2>&1;
 docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-osquery-close.yml > /dev/null 2>&1; 
 docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-ossec-close.yml > /dev/null 2>&1; 
 docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-strelka-close.yml > /dev/null 2>&1; 
 docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-syslog-close.yml > /dev/null 2>&1;
-{% for INDEX in THIRDPARTY.modules.keys() -%}
-docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-close.yml > /dev/null 2>&1;
-{% endfor -%}
-{% for INDEX in SO.modules.keys() -%}
+{% for INDEX in MODULESMERGED.modules.keys() -%}
 docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-close.yml > /dev/null 2>&1{% if not loop.last %};{% endif %}
 {% endfor -%}

salt/curator/files/bin/so-curator-closed-delete-delete

@@ -29,7 +29,7 @@ LOG="/opt/so/log/curator/so-curator-closed-delete.log"
 
 overlimit() {
 
-        [[ $(du -hs --block-size=1GB /nsm/elasticsearch/nodes | awk '{print $1}' ) -gt "{{LOG_SIZE_LIMIT}}" ]]
+        [[ $(du -hs --block-size=1GB /nsm/elasticsearch/indices | awk '{print $1}' ) -gt "{{LOG_SIZE_LIMIT}}" ]]
 }
 
 closedindices() {

salt/curator/files/bin/so-curator-cluster-close

@@ -24,21 +24,18 @@ read lastPID < $lf
 [ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit
 echo $$ > $lf
 
-{% from 'filebeat/map.jinja' import THIRDPARTY with context %}
-{% from 'filebeat/map.jinja' import SO with context %}
+{% from 'filebeat/modules.map.jinja' import MODULESMERGED with context %}
 
 docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-zeek-close.yml > /dev/null 2>&1; 
 docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-beats-close.yml > /dev/null 2>&1; 
 docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-firewall-close.yml > /dev/null 2>&1; 
 docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-ids-close.yml > /dev/null 2>&1; 
 docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-import-close.yml > /dev/null 2>&1; 
+docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-kratos-close.yml > /dev/null 2>&1;
 docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-osquery-close.yml > /dev/null 2>&1; 
 docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-ossec-close.yml > /dev/null 2>&1; 
 docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-strelka-close.yml > /dev/null 2>&1; 
 docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-syslog-close.yml > /dev/null 2>&1;
-{% for INDEX in THIRDPARTY.modules.keys() -%}
-docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-close.yml > /dev/null 2>&1;
-{% endfor -%}
-{% for INDEX in SO.modules.keys() -%}
+{% for INDEX in MODULESMERGED.modules.keys() -%}
 docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-close.yml > /dev/null 2>&1{% if not loop.last %};{% endif %}
 {% endfor -%}

salt/curator/files/bin/so-curator-cluster-delete

@@ -24,21 +24,18 @@ read lastPID < $lf
 [ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit
 echo $$ > $lf
 
-{% from 'filebeat/map.jinja' import THIRDPARTY with context %}
-{% from 'filebeat/map.jinja' import SO with context %}
+{% from 'filebeat/modules.map.jinja' import MODULESMERGED with context %}
 
 docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-zeek-delete.yml > /dev/null 2>&1; 
 docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-beats-delete.yml > /dev/null 2>&1; 
 docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-firewall-delete.yml > /dev/null 2>&1; 
 docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-ids-delete.yml > /dev/null 2>&1; 
-docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-import-delete.yml > /dev/null 2>&1; 
+docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-import-delete.yml > /dev/null 2>&1;
+docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-kratos-delete.yml > /dev/null 2>&1; 
 docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-osquery-delete.yml > /dev/null 2>&1; 
 docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-ossec-delete.yml > /dev/null 2>&1; 
 docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-strelka-delete.yml > /dev/null 2>&1; 
 docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-syslog-delete.yml > /dev/null 2>&1;
-{% for INDEX in THIRDPARTY.modules.keys() -%}
-docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-delete.yml > /dev/null 2>&1;
-{% endfor -%}
-{% for INDEX in SO.modules.keys() -%}
+{% for INDEX in MODULESMERGED.modules.keys() -%}
 docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-delete.yml > /dev/null 2>&1{% if not loop.last %};{% endif %}
 {% endfor -%}

salt/curator/files/bin/so-curator-cluster-warm

@@ -24,21 +24,18 @@ read lastPID < $lf
 [ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit
 echo $$ > $lf
 
-{% from 'filebeat/map.jinja' import THIRDPARTY with context %}
-{% from 'filebeat/map.jinja' import SO with context %}
+{% from 'filebeat/modules.map.jinja' import MODULESMERGED with context %}
 
 docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-zeek-warm.yml > /dev/null 2>&1; 
 docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-beats-warm.yml > /dev/null 2>&1; 
 docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-firewall-warm.yml > /dev/null 2>&1; 
 docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-ids-warm.yml > /dev/null 2>&1; 
-docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-import-warm.yml > /dev/null 2>&1; 
+docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-import-warm.yml > /dev/null 2>&1;
+docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-kratos-warm.yml > /dev/null 2>&1; 
 docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-osquery-warm.yml > /dev/null 2>&1; 
 docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-ossec-warm.yml > /dev/null 2>&1; 
 docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-strelka-warm.yml > /dev/null 2>&1; 
 docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-syslog-warm.yml > /dev/null 2>&1;
-{% for INDEX in THIRDPARTY.modules.keys() -%}
-docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-warm.yml > /dev/null 2>&1;
-{% endfor -%}
-{% for INDEX in SO.modules.keys() -%}
+{% for INDEX in MODULESMERGED.modules.keys() -%}
 docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-warm.yml > /dev/null 2>&1{% if not loop.last %};{% endif %}
 {% endfor -%}

salt/curator/init.sls

@@ -201,8 +201,8 @@ so-curatorclusterclose:
   cron.present:
     - name: /usr/sbin/so-curator-cluster-close > /opt/so/log/curator/cron-close.log 2>&1
     - user: root
-    - minute: '2'
-    - hour: '*/1'
+    - minute: '5'
+    - hour: '1'
     - daymonth: '*'
     - month: '*'
     - dayweek: '*'
@@ -211,8 +211,8 @@ so-curatorclusterdelete:
   cron.present:
     - name: /usr/sbin/so-curator-cluster-delete > /opt/so/log/curator/cron-delete.log 2>&1
     - user: root
-    - minute: '2'
-    - hour: '*/1'
+    - minute: '5'
+    - hour: '1'
     - daymonth: '*'
     - month: '*'
     - dayweek: '*'
@@ -221,8 +221,8 @@ so-curatorclusterwarm:
   cron.present:
     - name: /usr/sbin/so-curator-cluster-warm > /opt/so/log/curator/cron-warm.log 2>&1
     - user: root
-    - minute: '2'
-    - hour: '*/1'
+    - minute: '5'
+    - hour: '1'
     - daymonth: '*'
     - month: '*'
     - dayweek: '*'

salt/elastalert/init.sls

@@ -129,6 +129,9 @@ so-elastalert:
       - file: elastaconf
     - watch:
       - file: elastaconf
+    - onlyif:
+      - "so-elasticsearch-query / | jq -r '.version.number[0:1]' | grep -q 8" {# only run this state if elasticsearch is version 8 #}
+
 
 append_so-elastalert_so-status.conf:
   file.append:

salt/elasticsearch/defaults.yaml

@@ -53,11 +53,12 @@ elasticsearch:
     script:
       max_compilations_rate: 20000/1m
     indices:
-      query:
-        bool:
-          max_clause_count: 3500
       id_field_data:
         enabled: false
+    ingest:
+      geoip:
+        downloader:
+          enabled: false
     logger:
       org:
         elasticsearch:
@@ -4067,7 +4068,7 @@ elasticsearch:
                 field: "@timestamp"
                 order: desc
               refresh_interval: 30s
-              number_of_shards: 1
+              number_of_shards: 2
               number_of_replicas: 0
         composed_of: 
           - agent-mappings

salt/elasticsearch/files/ingest/filterlog

@@ -51,9 +51,10 @@
      },
      { "set":         { "field": "_index",        "value": "so-firewall", "override": true           } },
      { "set":         { "if": "ctx.network?.transport_id == '0'", "field": "network.transport",        "value": "icmp", "override": true           } },
-     {"community_id": {} },
-     { "set":         { "field": "module",        "value": "pfsense", "override": true           } },
-     { "set":         { "field": "dataset",        "value": "firewall", "override": true           } },
+     { "community_id": {} },
+     { "set":         { "field": "module",	"value": "pfsense",	"override": true	} },
+     { "set":         { "field": "dataset",	"value": "firewall",	"override": true	} },
+     { "set":         { "field": "category",	"value": "network",	"override": true	} },
      { "remove":      { "field": ["real_message", "ip_sub_msg", "firewall.sub_message"],     "ignore_failure": true  } }
   ]
 }

salt/elasticsearch/files/ingest/kratos

@@ -0,0 +1,13 @@
+{
+  "description" : "kratos",
+  "processors" : [
+    {
+      "set": {
+        "field": "_index",
+        "value": "so-kratos",
+        "override": true
+      }
+    },
+    { "pipeline":    { "name": "common" } }
+  ]
+}

salt/elasticsearch/files/ingest/rita.beacon

@@ -0,0 +1,127 @@
+{
+  "description": "RITA Beacons",
+  "processors": [
+    {
+      "set": {
+        "field": "_index",
+        "value": "so-rita",
+        "override": true
+      }
+    },
+    {
+      "csv": {
+        "field": "message",
+        "target_fields": [
+          "beacon.score",
+          "source.ip",
+          "destination.ip",
+          "network.connections",
+          "network.average_bytes",
+          "beacon.interval.range",
+          "beacon.size.range",
+          "beacon.interval.top",
+          "beacon.size.top",
+          "beacon.interval.top_count",
+          "beacon.size.top_count",
+          "beacon.interval.skew",
+          "beacon.size.skew",
+          "beacon.interval.dispersion",
+          "beacon.size.dispersion",
+          "network.bytes"
+        ]
+      }
+    },
+    {
+      "convert": {
+        "field": "beacon.score",
+        "type": "float"
+      }
+    },
+    {
+      "convert": {
+        "field": "network.connections",
+        "type": "integer"
+      }
+    },
+    {
+      "convert": {
+        "field": "network.average_bytes",
+        "type": "integer"
+      }
+    },
+    {
+      "convert": {
+        "field": "beacon.interval.range",
+        "type": "integer"
+      }
+    },
+    {
+      "convert": {
+        "field": "beacon.size.range",
+        "type": "integer"
+      }
+    },
+    {
+      "convert": {
+        "field": "beacon.interval.top",
+        "type": "integer"
+      }
+    },
+    {
+      "convert": {
+        "field": "beacon.size.top",
+        "type": "integer"
+      }
+    },
+    {
+      "convert": {
+        "field": "beacon.interval.top_count",
+        "type": "integer"
+      }
+    },
+    {
+      "convert": {
+        "field": "beacon.size.top_count",
+        "type": "integer"
+      }
+    },
+    {
+      "convert": {
+        "field": "beacon.interval.skew",
+        "type": "float"
+      }
+    },
+    {
+      "convert": {
+        "field": "beacon.size.skew",
+        "type": "float"
+      }
+    },
+    {
+      "convert": {
+        "field": "beacon.interval.dispersion",
+        "type": "integer"
+      }
+    },
+    {
+      "convert": {
+        "field": "beacon.size.dispersion",
+        "type": "integer"
+      }
+    },
+    {
+      "convert": {
+        "field": "network.bytes",
+        "type": "integer"
+      }
+    },
+    { "set":         { "if": "ctx.beacon?.score == 1", "field": "dataset",        "value": "alert", "override": true }},
+    { "set":         { "if": "ctx.beacon?.score == 1", "field": "rule.name",        "value": "Potential C2 Beacon Activity", "override": true }},
+    { "set":         { "if": "ctx.beacon?.score == 1", "field": "event.severity",        "value": 3, "override": true }},
+    {
+      "pipeline": {
+        "name": "common"
+      }
+    }
+  ]
+}

salt/elasticsearch/files/ingest/rita.connection

@@ -0,0 +1,36 @@
+{
+  "description": "RITA Connections",
+  "processors": [
+    {
+      "set": {
+        "field": "_index",
+        "value": "so-rita",
+        "override": true
+      }
+    },
+    {
+      "dissect": {
+        "field": "message",
+        "pattern": "%{source.ip},%{destination.ip},%{network.port}:%{network.protocol}:%{network.service},%{connection.duration},%{connection.state}"
+      }
+    },
+    {
+      "convert": {
+        "field": "connection.duration",
+        "type": "float"
+      }
+    },
+    {
+      "set": {
+        "field": "event.duration",
+        "value": "{{ connection.duration }}",
+        "override": true
+      }
+    },
+    {
+      "pipeline": {
+        "name": "common"
+      }
+    }
+  ]
+}

salt/elasticsearch/files/ingest/rita.dns

@@ -0,0 +1,39 @@
+{
+  "description": "RITA DNS",
+  "processors": [
+    {
+      "set": {
+        "field": "_index",
+        "value": "so-rita",
+        "override": true
+      }
+    },
+    {
+      "csv": {
+        "field": "message",
+        "target_fields": [
+          "dns.question.name",
+          "dns.question.subdomain_count",
+          "dns.question.count"
+        ]
+      }
+    },
+    {
+      "convert": {
+        "field": "dns.question.subdomain_count",
+        "type": "integer"
+      }
+    },
+    {
+      "convert": {
+        "field": "dns.question.count",
+        "type": "integer"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "common"
+      }
+    }
+  ]
+}

salt/elasticsearch/files/ingest/syslog

@@ -1,36 +1,157 @@
 {
-  "description" : "syslog",
+  "description" : "syslog pipeline",
   "processors" : [
     {
-        "dissect": {
-                "field": "message",
-                "pattern" : "%{message}",
-                "on_failure": [ { "drop" : { } } ]
-        },
-        "remove": {
-                "field": [ "type", "agent" ],
-                "ignore_failure": true
-        }
+      "dissect": {
+        "field": "message",
+        "pattern" : "%{message}",
+        "on_failure": [ { "drop" : { } } ]
+      },
+      "remove": {
+        "field": [ "type", "agent" ],
+        "ignore_failure": true
+      }
+    }, {
+      "grok": {
+        "field": "message",
+          "patterns": [
+            "^<%{INT:syslog.priority:int}>%{TIMESTAMP_ISO8601:syslog.timestamp} +%{IPORHOST:syslog.host} +%{PROG:syslog.program}(?:\\[%{POSINT:syslog.pid:int}\\])?: %{GREEDYDATA:real_message}$",
+
+            "^<%{INT:syslog.priority}>%{DATA:syslog.timestamp} %{WORD:source.application}(\\[%{DATA:pid}\\])?: %{GREEDYDATA:real_message}$",
+
+            "^%{SYSLOGTIMESTAMP:syslog.timestamp} %{SYSLOGHOST:syslog.host} %{SYSLOGPROG:syslog.program}: CEF:0\\|%{DATA:vendor}\\|%{DATA:product}\\|%{GREEDYDATA:message2}$"
+          ],
+          "ignore_failure": true
+      }
     },
     {
-        "grok":
-                {
-                        "field": "message",
-                        "patterns": [
-                                        "^<%{INT:syslog.priority}>%{DATA:syslog.timestamp} %{WORD:source.application}(\\[%{DATA:pid}\\])?: %{GREEDYDATA:real_message}$",
-                                        "^%{SYSLOGTIMESTAMP:syslog.timestamp} %{SYSLOGHOST:syslog.host} %{SYSLOGPROG:syslog.program}: CEF:0\\|%{DATA:vendor}\\|%{DATA:product}\\|%{GREEDYDATA:message2}$"
-                                    ],
-                        "ignore_failure": true
-                }
+      "convert" : {
+        "if": "ctx?.syslog?.priority != null",
+        "field" : "syslog.priority",
+        "type": "integer"
+      }
     },
-    { "set": { "if": "ctx.source?.application == 'filterlog'", "field": "dataset", "value": "firewall", "ignore_failure": true  } },
-    { "set": { "if": "ctx.vendor != null", "field": "module", "value": "{{ vendor }}", "ignore_failure": true } },
-    { "set": { "if": "ctx.product != null", "field": "dataset", "value": "{{ product }}", "ignore_failure": true } },
-    { "set": { "field": "event.ingested",      "value": "{{ @timestamp }}"       } },
-    { "date": { "if": "ctx.syslog?.timestamp != null", "field": "syslog.timestamp",       "target_field": "@timestamp",   "formats": ["MMM  d HH:mm:ss", "MMM dd HH:mm:ss", "ISO8601", "UNIX"], "ignore_failure": true  } },
-    { "remove":         { "field": ["pid", "program"],  "ignore_missing": true, "ignore_failure": true  } },
-    { "pipeline": { "if": "ctx.vendor != null && ctx.product != null", "name": "{{ vendor }}.{{ product }}", "ignore_failure": true } },
-    { "pipeline": { "if": "ctx.dataset == 'firewall'", "name": "filterlog", "ignore_failure": true } },
-    { "pipeline":       { "name": "common"                                             } }
+    {
+      "script": {
+        "description": "Map syslog priority into facility and level",
+        "lang": "painless",
+        "params" : {
+          "level": [
+            "emerg",
+            "alert",
+            "crit",
+            "err",
+            "warn",
+            "notice",
+            "info",
+            "debug"
+           ],
+           "facility" : [
+             "kern",
+             "user",
+             "mail",
+             "daemon",
+             "auth",
+             "syslog",
+             "lpr",
+             "news",
+             "uucp",
+             "cron",
+             "authpriv",
+             "ftp",
+             "ntp",
+             "security",
+             "console",
+             "solaris-cron",
+             "local0",
+             "local1",
+             "local2",
+             "local3",
+             "local4",
+             "local5",
+             "local6",
+             "local7"
+            ]
+        },
+        "source": "if (ctx['syslog'] != null && ctx['syslog']['priority'] != null) { int p = ctx['syslog']['priority']; int f = p / 8; int l = p - (f * 8); ctx['syslog']['facility_label'] = [ : ]; ctx['syslog']['severity_label'] = [ : ]; ctx['syslog'].put('severity', l); ctx['syslog'].put('severity_label', params.level[l].toUpperCase()); ctx['syslog'].put('facility', f); ctx['syslog'].put('facility_label', params.facility[f].toUpperCase()); }"
+
+      }
+    },
+    {
+      "set": {
+        "if": "ctx.syslog?.host != null",
+        "field": "host.name",
+        "value": "{{ syslog.host }}",
+        "ignore_failure": true
+      }
+    }, {
+      "set": {
+        "if": "ctx.syslog?.program != null",
+        "field": "process.name",
+        "value": "{{ syslog.program }}",
+        "ignore_failure": true
+      }
+    }, {
+      "set": {
+        "if": "ctx.syslog?.pid != null",
+        "field": "process.id",
+        "value": "{{ syslog.pid }}",
+        "ignore_failure": true
+      }
+    }, {
+      "set": {
+        "if": "ctx.source?.application == 'filterlog'",
+        "field": "dataset",
+        "value": "firewall",
+        "ignore_failure": true
+      }
+    }, {
+      "set": {
+        "if": "ctx.vendor != null",
+        "field": "module",
+        "value": "{{ vendor }}",
+        "ignore_failure": true
+      }
+    }, {
+      "set": {
+        "if": "ctx.product != null",
+        "field": "dataset",
+        "value": "{{ product }}",
+        "ignore_failure": true
+      }
+    }, {
+      "set": {
+        "field": "ingest.timestamp",
+        "value": "{{ @timestamp }}"
+      }
+    }, {
+      "date": {
+        "if": "ctx.syslog?.timestamp != null",
+        "field": "syslog.timestamp",
+        "target_field": "@timestamp",
+        "formats": ["MMM  d HH:mm:ss", "MMM dd HH:mm:ss", "ISO8601", "UNIX"],
+        "ignore_failure": true
+      }
+    }, {
+      "remove": {
+        "field": ["pid", "program"],
+        "ignore_missing": true,
+        "ignore_failure": true
+      }
+    }, {
+      "pipeline": {
+        "if": "ctx.vendor != null && ctx.product != null",
+        "name": "{{ vendor }}.{{ product }}",
+        "ignore_failure": true
+      }
+    }, {
+      "pipeline": {
+        "if": "ctx.dataset == 'firewall'",
+        "name": "filterlog",
+        "ignore_failure": true
+      }
+    }, {
+      "pipeline": { "name": "common" }
+    }
   ]
 }

salt/elasticsearch/files/ingest/sysmon

@@ -9,61 +9,70 @@
     { "set":           { "if": "ctx.event?.code == '5'",   "field": "event.category", "value": "host,process", "override": true }  },
     { "set":           { "if": "ctx.event?.code == '6'",   "field": "event.category", "value": "host,driver", "override": true }  },
     { "set":           { "if": "ctx.event?.code == '22'",   "field": "event.category", "value": "network", "override": true }  },
-    { "set":          { "if": "ctx.event?.code == '1'",   "field": "event.dataset", "value": "process_creation", "override": true }  },
-    { "set":          { "if": "ctx.event?.code == '2'",   "field": "event.dataset", "value": "process_changed_file", "override": true }  },
-    { "set":          { "if": "ctx.event?.code == '3'",   "field": "event.dataset", "value": "network_connection", "override": true }  },
-    { "set":          { "if": "ctx.event?.code == '5'",   "field": "event.dataset", "value": "process_terminated", "override": true }  },
-    { "set":          { "if": "ctx.event?.code == '6'",   "field": "event.dataset", "value": "driver_loaded", "override": true }  },
-    { "set":          { "if": "ctx.event?.code == '7'",   "field": "event.dataset", "value": "image_loaded", "override": true }  },
-    { "set":          { "if": "ctx.event?.code == '8'",   "field": "event.dataset", "value": "create_remote_thread", "override": true }  },
-    { "set":          { "if": "ctx.event?.code == '9'",   "field": "event.dataset", "value": "raw_file_access_read", "override": true }  },
-    { "set":          { "if": "ctx.event?.code == '10'",   "field": "event.dataset", "value": "process_access", "override": true }  },
-    { "set":          { "if": "ctx.event?.code == '11'",   "field": "event.dataset", "value": "file_create", "override": true }  },
-    { "set":          { "if": "ctx.event?.code == '12'",   "field": "event.dataset", "value": "registry_create_delete", "override": true }  },
-    { "set":          { "if": "ctx.event?.code == '13'",   "field": "event.dataset", "value": "registry_value_set", "override": true }  },
-    { "set":          { "if": "ctx.event?.code == '14'",   "field": "event.dataset", "value": "registry_key_value_rename", "override": true }  },
-    { "set":          { "if": "ctx.event?.code == '15'",   "field": "event.dataset", "value": "file_create_stream_hash", "override": true }  },
-    { "set":          { "if": "ctx.event?.code == '16'",   "field": "event.dataset", "value": "config_change", "override": true }  },
-    { "set":          { "if": "ctx.event?.code == '22'",   "field": "event.dataset", "value": "dns_query", "override": true }  },
-    { "rename": 	   { "field": "winlog.event_data.SubjectUserName", 			  "target_field": "user.name",		"ignore_missing": true 	} },
-    { "rename": 	   { "field": "winlog.event_data.DestinationHostname", 	  "target_field": "destination.hostname",	"ignore_missing": true 	} },
-    { "rename": 	   { "field": "winlog.event_data.DestinationIp", 		      "target_field": "destination.ip",	"ignore_missing": true 	} },
-    { "rename": 	   { "field": "winlog.event_data.DestinationPort", 	      "target_field": "destination.port",	"ignore_missing": true 	} },
-    { "rename":      { "field": "winlog.event_data.image",                  "target_field": "process.executable",               "ignore_missing": true  } }, 
-    { "rename": 	   { "field": "winlog.event_data.Image", 			            "target_field": "process.executable",		"ignore_missing": true 	} },
-    { "rename":      { "field": "winlog.event_data.processID",              "target_field": "process.pid",          "ignore_missing": true  } }, 
-    { "rename": 	   { "field": "winlog.event_data.ProcessId", 			        "target_field": "process.pid",		"ignore_missing": true 	} },
-    { "rename":      { "field": "winlog.event_data.processGuid",            "target_field": "process.entity_id",              "ignore_missing": true  } },
-    { "rename": 	   { "field": "winlog.event_data.ProcessGuid", 			      "target_field": "process.entity_id",		"ignore_missing": true 	} },
-    { "rename":      { "field": "winlog.event_data.commandLine",            "target_field": "process.command_line",           "ignore_missing": true  } },    
-    { "rename": 	   { "field": "winlog.event_data.CommandLine", 			      "target_field": "process.command_line",		"ignore_missing": true 	} },
-    { "rename":      { "field": "winlog.event_data.currentDirectory",       "target_field": "process.working_directory",            "ignore_missing": true  } },
-    { "rename": 	   { "field": "winlog.event_data.CurrentDirectory", 			"target_field": "process.working_directory",		"ignore_missing": true 	} },
-    { "rename":      { "field": "winlog.event_data.description",            "target_field": "process.pe.description",         "ignore_missing": true  } },
-    { "rename": 	   { "field": "winlog.event_data.Description", 			      "target_field": "process.pe.description",		"ignore_missing": true 	} },
-    { "rename":      { "field": "winlog.event_data.product",                "target_field": "process.pe.product",         "ignore_missing": true  } },
-    { "rename": 	   { "field": "winlog.event_data.Product", 			          "target_field": "process.pe.product",		"ignore_missing": true 	} },
-    { "rename":      { "field": "winlog.event_data.company",                "target_field": "process.pe.company",         "ignore_missing": true  } }, 
-    { "rename": 	   { "field": "winlog.event_data.Company", 			          "target_field": "process.pe.company",		"ignore_missing": true 	} },
-    { "rename":      { "field": "winlog.event_data.originalFileName",       "target_field": "process.pe.original_file_name",                "ignore_missing": true  } }, 
-    { "rename": 	   { "field": "winlog.event_data.OriginalFileName", 			"target_field": "process.pe.original_file_name",		"ignore_missing": true 	} },
-    { "rename":      { "field": "winlog.event_data.fileVersion",            "target_field": "process.pe.file_version",                "ignore_missing": true  } },
-    { "rename": 	   { "field": "winlog.event_data.FileVersion", 			      "target_field": "process.pe.file_version",		"ignore_missing": true 	} },
-    { "rename":      { "field": "winlog.event_data.parentCommandLine",      "target_field": "process.parent.command_line",          "ignore_missing": true  } },
-    { "rename": 	   { "field": "winlog.event_data.ParentCommandLine", 			"target_field": "process.parent.command_line",		"ignore_missing": true 	} },
-    { "rename":      { "field": "winlog.event_data.parentImage",            "target_field": "process.parent.executable",              "ignore_missing": true  } },
-    { "rename": 	   { "field": "winlog.event_data.ParentImage", 			      "target_field": "process.parent.executable",		"ignore_missing": true 	} },
-    { "rename":      { "field": "winlog.event_data.parentProcessGuid",      "target_field": "process.parent.entity_id",             "ignore_missing": true  } }, 
-    { "rename": 	   { "field": "winlog.event_data.ParentProcessGuid", 			"target_field": "process.parent.entity_id",		"ignore_missing": true 	} },
-    { "rename":      { "field": "winlog.event_data.parentProcessId",        "target_field": "process.ppid",               "ignore_missing": true  } },
-    { "rename": 	   { "field": "winlog.event_data.ParentProcessId", 			  "target_field": "process.ppid",		"ignore_missing": true 	} },
-    { "rename": 	   { "field": "winlog.event_data.Protocol", 	            "target_field": "network.transport",	"ignore_missing": true 	} },
-    { "rename": 	   { "field": "winlog.event_data.User", 			            "target_field": "user.name",		"ignore_missing": true 	} },
-    { "rename": 	   { "field": "winlog.event_data.SourceHostname", 	      "target_field": "source.hostname",	"ignore_missing": true 	} },
-    { "rename": 	   { "field": "winlog.event_data.SourceIp", 		          "target_field": "source.ip",		"ignore_missing": true 	} },
-    { "rename": 	   { "field": "winlog.event_data.SourcePort", 		        "target_field": "source.port",		"ignore_missing": true 	} },   
-    { "rename": 	   { "field": "winlog.event_data.targetFilename",		      "target_field": "file.target",	"ignore_missing": true 	} },
-    { "rename":      { "field": "winlog.event_data.TargetFilename",         "target_field": "file.target",    "ignore_missing": true  } },
+    { "set":           { "if": "ctx.event?.code == '1'",   "field": "event.dataset", "value": "process_creation", "override": true }  },
+    { "set":           { "if": "ctx.event?.code == '2'",   "field": "event.dataset", "value": "process_changed_file", "override": true }  },
+    { "set":           { "if": "ctx.event?.code == '3'",   "field": "event.dataset", "value": "network_connection", "override": true }  },
+    { "set":           { "if": "ctx.event?.code == '5'",   "field": "event.dataset", "value": "process_terminated", "override": true }  },
+    { "set":           { "if": "ctx.event?.code == '6'",   "field": "event.dataset", "value": "driver_loaded", "override": true }  },
+    { "set":           { "if": "ctx.event?.code == '7'",   "field": "event.dataset", "value": "image_loaded", "override": true }  },
+    { "set":           { "if": "ctx.event?.code == '8'",   "field": "event.dataset", "value": "create_remote_thread", "override": true }  },
+    { "set":           { "if": "ctx.event?.code == '9'",   "field": "event.dataset", "value": "raw_file_access_read", "override": true }  },
+    { "set":           { "if": "ctx.event?.code == '10'",   "field": "event.dataset", "value": "process_access", "override": true }  },
+    { "set":           { "if": "ctx.event?.code == '11'",   "field": "event.dataset", "value": "file_create", "override": true }  },
+    { "set":           { "if": "ctx.event?.code == '12'",   "field": "event.dataset", "value": "registry_create_delete", "override": true }  },
+    { "set":           { "if": "ctx.event?.code == '13'",   "field": "event.dataset", "value": "registry_value_set", "override": true }  },
+    { "set":           { "if": "ctx.event?.code == '14'",   "field": "event.dataset", "value": "registry_key_value_rename", "override": true }  },
+    { "set":           { "if": "ctx.event?.code == '15'",   "field": "event.dataset", "value": "file_create_stream_hash", "override": true }  },
+    { "set":           { "if": "ctx.event?.code == '16'",   "field": "event.dataset", "value": "config_change", "override": true }  },
+    { "set":           { "if": "ctx.event?.code == '22'",   "field": "event.dataset", "value": "dns_query", "override": true }  },
+    { "kv":            {"field":  "winlog.event_data.Hashes",  "target_field": "file.hash",    "field_split": ",",     "value_split": "=",     "ignore_missing": true } },
+    { "kv":            {"field":  "winlog.event_data.Hash",  "target_field": "file.hash",    "field_split": ",",     "value_split": "=",     "ignore_missing": true } },
+    { "rename":      { "field": "file.hash.IMPHASH", "target_field": "hash.imphash",	"ignore_missing":true } },
+    { "rename":      { "field": "file.hash.MD5", "target_field": "hash.md5",    "ignore_missing":true } },
+    { "rename":      { "field": "file.hash.SHA256", "target_field": "hash.sha256",    "ignore_missing":true } },
+    { "rename":      { "field": "winlog.event_data.SubjectUserName",	"target_field": "user.name",		"ignore_missing": true 	} },
+    { "rename":      { "field": "winlog.event_data.DestinationHostname",	"target_field": "destination.hostname",	"ignore_missing": true 	} },
+    { "rename":      { "field": "winlog.event_data.DestinationIp",	"target_field": "destination.ip",	"ignore_missing": true 	} },
+    { "rename":      { "field": "winlog.event_data.DestinationPort",	"target_field": "destination.port",	"ignore_missing": true 	} },
+    { "rename":      { "field": "winlog.event_data.image", 		"target_field": "process.executable",               "ignore_missing": true  } }, 
+    { "rename":      { "field": "winlog.event_data.Image", 		"target_field": "process.executable",		"ignore_missing": true 	} },
+    { "rename":      { "field": "winlog.event_data.processID",          "target_field": "process.pid",          "ignore_missing": true  } }, 
+    { "rename":      { "field": "winlog.event_data.ProcessId", 		"target_field": "process.pid",		"ignore_missing": true 	} },
+    { "rename":      { "field": "winlog.event_data.processGuid",        "target_field": "process.entity_id",              "ignore_missing": true  } },
+    { "rename":      { "field": "winlog.event_data.ProcessGuid", 	"target_field": "process.entity_id",		"ignore_missing": true 	} },
+    { "rename":      { "field": "winlog.event_data.commandLine",        "target_field": "process.command_line",           "ignore_missing": true  } },    
+    { "rename":      { "field": "winlog.event_data.CommandLine", 	"target_field": "process.command_line",		"ignore_missing": true 	} },
+    { "rename":      { "field": "winlog.event_data.currentDirectory",   "target_field": "process.working_directory",            "ignore_missing": true  } },
+    { "rename":      { "field": "winlog.event_data.CurrentDirectory", 	"target_field": "process.working_directory",		"ignore_missing": true 	} },
+    { "rename":      { "field": "winlog.event_data.description",        "target_field": "process.pe.description",         "ignore_missing": true  } },
+    { "rename":      { "field": "winlog.event_data.Description", 	"target_field": "process.pe.description",		"ignore_missing": true 	} },
+    { "rename":      { "field": "winlog.event_data.product",            "target_field": "process.pe.product",         "ignore_missing": true  } },
+    { "rename":      { "field": "winlog.event_data.Product", 		"target_field": "process.pe.product",		"ignore_missing": true 	} },
+    { "rename":      { "field": "winlog.event_data.company",            "target_field": "process.pe.company",         "ignore_missing": true  } }, 
+    { "rename":      { "field": "winlog.event_data.Company", 		"target_field": "process.pe.company",		"ignore_missing": true 	} },
+    { "rename":      { "field": "winlog.event_data.originalFileName",   "target_field": "process.pe.original_file_name",                "ignore_missing": true  } }, 
+    { "rename":      { "field": "winlog.event_data.OriginalFileName", 	"target_field": "process.pe.original_file_name",		"ignore_missing": true 	} },
+    { "rename":      { "field": "winlog.event_data.fileVersion",        "target_field": "process.pe.file_version",                "ignore_missing": true  } },
+    { "rename":      { "field": "winlog.event_data.FileVersion", 	"target_field": "process.pe.file_version",		"ignore_missing": true 	} },
+    { "rename":      { "field": "winlog.event_data.parentCommandLine",  "target_field": "process.parent.command_line",          "ignore_missing": true  } },
+    { "rename":      { "field": "winlog.event_data.ParentCommandLine", 	"target_field": "process.parent.command_line",		"ignore_missing": true 	} },
+    { "rename":      { "field": "winlog.event_data.parentImage",        "target_field": "process.parent.executable",              "ignore_missing": true  } },
+    { "rename":      { "field": "winlog.event_data.ParentImage", 	"target_field": "process.parent.executable",		"ignore_missing": true 	} },
+    { "rename":      { "field": "winlog.event_data.parentProcessGuid",  "target_field": "process.parent.entity_id",             "ignore_missing": true  } }, 
+    { "rename":      { "field": "winlog.event_data.ParentProcessGuid", 	"target_field": "process.parent.entity_id",		"ignore_missing": true 	} },
+    { "rename":      { "field": "winlog.event_data.parentProcessId",    "target_field": "process.ppid",               "ignore_missing": true  } },
+    { "rename":      { "field": "winlog.event_data.ParentProcessId", 	"target_field": "process.ppid",		"ignore_missing": true 	} },
+    { "rename":      { "field": "winlog.event_data.Protocol", 	        "target_field": "network.transport",	"ignore_missing": true 	} },
+    { "rename":      { "field": "winlog.event_data.User", 		"target_field": "user.name",		"ignore_missing": true 	} },
+    { "rename":      { "field": "winlog.event_data.SourceHostname", 	"target_field": "source.hostname",	"ignore_missing": true 	} },
+    { "rename":      { "field": "winlog.event_data.SourceIp", 		"target_field": "source.ip",		"ignore_missing": true 	} },
+    { "rename":      { "field": "winlog.event_data.SourcePort", 	"target_field": "source.port",		"ignore_missing": true 	} },   
+    { "rename":      { "field": "winlog.event_data.targetFilename",	"target_field": "file.target",	"ignore_missing": true 	} },
+    { "rename":      { "field": "winlog.event_data.TargetFilename",     "target_field": "file.target",    "ignore_missing": true  } },
+    { "rename":      { "field": "winlog.event_data.QueryResults",       "target_field": "dns.answers.name",    "ignore_missing": true  } },
+    { "rename":      { "field": "winlog.event_data.QueryName",          "target_field": "dns.query.name",    "ignore_missing": true  } },
+    { "remove":      { "field": "winlog.event_data.Hash",		"ignore_missing": true  } },
+    { "remove":      { "field": "winlog.event_data.Hashes",		"ignore_missing": true  } },
     { "community_id": {} }
   ]
 }

salt/elasticsearch/files/ingest/zeek.bacnet

@@ -0,0 +1,14 @@
+{
+  "description" : "zeek.bacnet",
+  "processors" : [
+    { "remove":         { "field": ["host"],								"ignore_failure": true	} },
+    { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
+    { "rename": 	{ "field": "message2.is_orig",		"target_field": "bacnet.is_orig",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.bvlc_function",	"target_field": "bacnet.bclv.function",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.pdu_type",		"target_field": "bacnet.pdu.type",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.pdu_service",	"target_field": "bacnet.pdu.service",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.invoke_id",	"target_field": "bacnet.invoke.id",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.result_code",	"target_field": "bacnet.result.code",	"ignore_missing": true 	} },
+    { "pipeline":       { "name": "zeek.common" } }
+  ]
+}

salt/elasticsearch/files/ingest/zeek.bacnet_discovery

@@ -0,0 +1,15 @@
+{
+  "description" : "zeek.bacnet_discovery",
+  "processors" : [
+    { "remove":         { "field": ["host"],									"ignore_failure": true	} },
+    { "json":		{ "field": "message",			"target_field": "message2",			"ignore_failure": true	} },
+    { "rename":         { "field": "message2.is_orig",		"target_field": "bacnet.is_orig",		"ignore_missing": true  } },
+    { "rename":         { "field": "message2.pdu_service",	"target_field": "bacnet.pdu.service",		"ignore_missing": true  } },
+    { "rename":         { "field": "message2.object_type",	"target_field": "bacnet.object.type",		"ignore_missing": true  } },
+    { "rename":         { "field": "message2.instance_number",	"target_field": "bacnet.instance.number",	"ignore_missing": true  } },
+    { "rename":         { "field": "message2.vendor",		"target_field": "bacnet.vendor",		"ignore_missing": true  } },
+    { "rename":         { "field": "message2.range",		"target_field": "bacnet.range",			"ignore_missing": true  } },
+    { "rename":         { "field": "message2.object_name",	"target_field": "bacnet.object.name",		"ignore_missing": true  } },
+    { "pipeline":       { "name": "zeek.common" } }
+  ]
+}

salt/elasticsearch/files/ingest/zeek.bacnet_property

@@ -0,0 +1,15 @@
+{
+  "description" : "zeek.bacnet_property",
+  "processors" : [
+    { "remove":         { "field": ["host"],									"ignore_failure": true	} },
+    { "json":           { "field": "message",			"target_field": "message2",			"ignore_failure": true  } },
+    { "rename":         { "field": "message2.is_orig",          "target_field": "bacnet.is_orig",         "ignore_missing": true  } },
+    { "rename":         { "field": "message2.instance_number",	"target_field": "bacnet.instance.number",	"ignore_missing": true  } },
+    { "rename":         { "field": "message2.pdu_service",	"target_field": "bacnet.pdu.service",		"ignore_missing": true  } },
+    { "rename":         { "field": "message2.object_type",	"target_field": "bacnet.object.type",		"ignore_missing": true  } },
+    { "rename":         { "field": "message2.property",		"target_field": "bacnet.property",		"ignore_missing": true  } },
+    { "rename":         { "field": "message2.array_index",	"target_field": "bacnet.array.index",		"ignore_missing": true  } },
+    { "rename":         { "field": "message2.value",		"target_field": "bacnet.value",			"ignore_missing": true  } },
+    { "pipeline":       { "name": "zeek.common" } }
+  ]
+}

salt/elasticsearch/files/ingest/zeek.bsap_ip_header

@@ -0,0 +1,10 @@
+{
+  "description" : "zeek.bsap_ip_header",
+  "processors" : [
+    { "remove":         { "field": ["host"],									"ignore_failure": true	} },
+    { "json":		{ "field": "message",			"target_field": "message2",			"ignore_failure": true	} },
+    { "rename": 	{ "field": "message2.num_msg",		"target_field": "bsap.number.messages",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.type_name",	"target_field": "bsap.message.type",		"ignore_missing": true 	} },
+    { "pipeline":       { "name": "zeek.common" } }
+  ]
+}

salt/elasticsearch/files/ingest/zeek.bsap_ip_rdb

@@ -0,0 +1,20 @@
+{
+  "description" : "zeek.bsap_ip_rdb",
+  "processors" : [
+    { "remove":         { "field": ["host"],										"ignore_failure": true	} },
+    { "json":		{ "field": "message",			"target_field": "message2",				"ignore_failure": true	} },
+    { "rename": 	{ "field": "message2.header_size",	"target_field": "bsap.header.length",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.mes_seq",		"target_field": "bsap.message.sequence",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.res_seq",		"target_field": "bsap.response.sequence",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.data_len",		"target_field": "bsap.data.length",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.sequence",		"target_field": "bsap.function.sequence",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.app_func_code",	"target_field": "bsap.application.function",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.node_status",	"target_field": "bsap.node.status",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.func_code",	"target_field": "bsap.application.sub_function",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.variable_count",	"target_field": "bsap.variable.count",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.variables",	"target_field": "bsap.vector.variables",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.variable_value",	"target_field": "bsap.vector.variable.value",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.value",		"target_field": "bsap.value",				"ignore_missing": true 	} },
+    { "pipeline":       { "name": "zeek.common" } }
+  ]
+}

salt/elasticsearch/files/ingest/zeek.bsap_ip_unknown

@@ -0,0 +1,9 @@
+{
+  "description" : "zeek.bsap_ip_unknown",
+  "processors" : [
+    { "remove":         { "field": ["host"],							"ignore_failure": true	} },
+    { "json":		{ "field": "message",		"target_field": "message2",		"ignore_failure": true	} },
+    { "rename": 	{ "field": "message2.data",	"target_field": "bsap.ip.unknown.data",	"ignore_missing": true 	} },
+    { "pipeline":       { "name": "zeek.common" } }
+  ]
+}

salt/elasticsearch/files/ingest/zeek.bsap_serial_header

@@ -0,0 +1,17 @@
+{
+  "description" : "zeek.bsap_serial_header",
+  "processors" : [
+    { "remove":         { "field": ["host"],									"ignore_failure": true	} },
+    { "json":		{ "field": "message",			"target_field": "message2",			"ignore_failure": true	} },
+    { "rename": 	{ "field": "message2.ser",		"target_field": "bsap.message.serial_number",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.dadd",		"target_field": "bsap.destination.address",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.sadd",		"target_field": "bsap.source.address",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.ctl",		"target_field": "bsap.control.byte",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.dfun",		"target_field": "bsap.destination.function",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.seq",		"target_field": "bsap.message.sequence",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.sfun",		"target_field": "bsap.source.function",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.nsb",		"target_field": "bsap.node.status_byte",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.type_name",	"target_field": "bsap.message.type",		"ignore_missing": true 	} },
+    { "pipeline":       { "name": "zeek.common" } }
+  ]
+}

salt/elasticsearch/files/ingest/zeek.bsap_serial_rdb

@@ -0,0 +1,11 @@
+{
+  "description" : "zeek.bsap_serial_rdb",
+  "processors" : [
+    { "remove":         { "field": ["host"],									"ignore_failure": true  } },
+    { "json":		{ "field": "message",			"target_field": "message2",			"ignore_failure": true	} },
+    { "rename": 	{ "field": "message2.func_code",	"target_field": "bsap.rdb.function",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.variables",	"target_field": "bsap.vector.variables",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.variable_value",	"target_field": "bsap.vector.value",		"ignore_missing": true 	} },
+    { "pipeline":       { "name": "zeek.common" } }
+  ]
+}

salt/elasticsearch/files/ingest/zeek.bsap_serial_rdb_ext

@@ -0,0 +1,13 @@
+{
+  "description" : "zeek.bsap_serial_rdb_ext",
+  "processors" : [
+    { "remove":         { "field": ["host"],								"ignore_failure": true  } },
+    { "json":		{ "field": "message",		"target_field": "message2",			"ignore_failure": true	} },
+    { "rename": 	{ "field": "message2.dfun",	"target_field": "bsap.destination.function",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.seq",	"target_field": "bsap.message_sequence",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.nsb",	"target_field": "bsap.node_status_byte",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.extfun",	"target_field": "bsap.extension.function",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.data",	"target_field": "bsap.extension.function_data",	"ignore_missing": true 	} },
+    { "pipeline":       { "name": "zeek.common" } }
+  ]
+}

salt/elasticsearch/files/ingest/zeek.bsap_serial_unknown

@@ -0,0 +1,9 @@
+{
+  "description" : "zeek.bsap_serial_unknown",
+  "processors" : [
+    { "remove":         { "field": ["host"],								"ignore_failure": true  } },
+    { "json":		{ "field": "message",		"target_field": "message2",			"ignore_failure": true	} },
+    { "rename": 	{ "field": "message2.data",	"target_field": "bsap.serial.unknown.data",	"ignore_missing": true 	} },
+    { "pipeline":       { "name": "zeek.common" } }
+  ]
+}

salt/elasticsearch/files/ingest/zeek.cip

@@ -0,0 +1,19 @@
+{
+  "description" : "zeek.cip",
+  "processors" : [
+    { "remove":         { "field": ["host"],											"ignore_failure": true	} },
+    { "json":		{ "field": "message",				"target_field": "message2",				"ignore_failure": true	} },
+    { "rename": 	{ "field": "message2.is_orig",			"target_field": "cip.is_orig",			        "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.cip_sequence_count",	"target_field": "cip.sequence_count",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.direction",		"target_field": "cip.direction",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.cip_service_code",		"target_field": "cip.service_code",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.cip_service",		"target_field": "cip.service",				"ignore_missing": true 	} },
+    { "convert":	{ "field": "cip.service",			"type": "string",					"ignore_missing": true  } },
+    { "rename": 	{ "field": "message2.cip_status",		"target_field": "cip.status_code",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.class_id",			"target_field": "cip.request.path.class.id",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.class_name",		"target_field": "cip.request.path.class.name",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.instance_id",		"target_field": "cip.request.path.instance.id",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.attribute_id",		"target_field": "cip.request.path.attribute.id",	"ignore_missing": true 	} },
+    { "pipeline":       { "name": "zeek.common" } }
+  ]
+}

salt/elasticsearch/files/ingest/zeek.cip_identity

@@ -0,0 +1,21 @@
+{
+  "description" : "zeek.cip_identity",
+  "processors" : [
+    { "remove":         { "field": ["host"],										"ignore_failure": true	} },
+    { "json":		{ "field": "message",				"target_field": "message2",			"ignore_failure": true	} },
+    { "rename": 	{ "field": "message2.encapsulation_version",	"target_field": "cip.encapsulation.version",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.socket_address",		"target_field": "cip.socket.address",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.socket_port",		"target_field": "cip.socket.port",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.vendor_id",		"target_field": "cip.vendor.id",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.vendor_name",		"target_field": "cip.vendor.name",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.device_type_id",		"target_field": "cip.device.type.id",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.device_type_name",		"target_field": "cip.device.type.name",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.product_code",		"target_field": "cip.device.product.code",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.revision",			"target_field": "cip.device.revision",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.device_status",		"target_field": "cip.device.status",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.serial_number",		"target_field": "cip.device.serial_number",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.product_name",		"target_field": "cip.device.product.name",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.device_state",		"target_field": "cip.device.state",		"ignore_missing": true 	} },
+    { "pipeline":       { "name": "zeek.common" } }
+  ]
+}

salt/elasticsearch/files/ingest/zeek.cip_io

@@ -0,0 +1,13 @@
+{
+  "description" : "zeek.cip_io",
+  "processors" : [
+    { "remove":         { "field": ["host"],								"ignore_failure": true	} },
+    { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
+    { "rename": 	{ "field": "message2.is_orig",		"target_field": "cip.is_orig",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.connection_id",	"target_field": "cip.connection.id",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.sequence_number",	"target_field": "cip.sequence_number",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.data_length",	"target_field": "cip.data.length",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.io_data",		"target_field": "cip.io.data",		"ignore_missing": true 	} },
+    { "pipeline":       { "name": "zeek.common"	} }
+  ]
+}

salt/elasticsearch/files/ingest/zeek.conn

@@ -17,24 +17,25 @@
     { "rename": 	{ "field": "message2.orig_ip_bytes", 	"target_field": "client.ip_bytes",	"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.resp_pkts", 	"target_field": "server.packets",	"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.resp_ip_bytes", 	"target_field": "server.ip_bytes",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.orig_mac_oui", 	"target_field": "client.oui",	"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.tunnel_parents",	"target_field": "log.id.tunnel_parents",	"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.orig_cc", 		"target_field": "client.country_code","ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.resp_cc", 		"target_field": "server.country_code",	"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.sensorname", 	"target_field": "observer.name",		"ignore_missing": true 	} },
     { "script":         { "lang": "painless", "source": "ctx.network.bytes = (ctx.client.bytes + ctx.server.bytes)", "ignore_failure": true } },
-    { "set": { "if": "ctx.connection.state == 'S0'",    "field": "connection.state_description", "value": "Connection attempt seen, no reply" } },
-    { "set": { "if": "ctx.connection.state == 'S1'",    "field": "connection.state_description", "value": "Connection established, not terminated" } },
-    { "set": { "if": "ctx.connection.state == 'S2'",    "field": "connection.state_description", "value": "Connection established and close attempt by originator seen (but no reply from responder)" } },
-    { "set": { "if": "ctx.connection.state == 'S3'",    "field": "connection.state_description", "value": "Connection established and close attempt by responder seen (but no reply from originator)" } },
-    { "set": { "if": "ctx.connection.state == 'SF'",    "field": "connection.state_description", "value": "Normal SYN/FIN completion" } },
-    { "set": { "if": "ctx.connection.state == 'REJ'",   "field": "connection.state_description", "value": "Connection attempt rejected" } },
-    { "set": { "if": "ctx.connection.state == 'RSTO'",  "field": "connection.state_description", "value": "Connection established, originator aborted (sent a RST)" } },
-    { "set": { "if": "ctx.connection.state == 'RSTR'",  "field": "connection.state_description", "value": "Established, responder aborted" } },
-    { "set": { "if": "ctx.connection.state == 'RSTOS0'","field": "connection.state_description", "value": "Originator sent a SYN followed by a RST, we never saw a SYN-ACK from the responder" } },
-    { "set": { "if": "ctx.connection.state == 'RSTRH'", "field": "connection.state_description", "value": "Responder sent a SYN ACK followed by a RST, we never saw a SYN from the (purported) originator" } },
-    { "set": { "if": "ctx.connection.state == 'SH'",    "field": "connection.state_description", "value": "Originator sent a SYN followed by a FIN, we never saw a SYN ACK from the responder (hence the connection was 'half' open)" } },
-    { "set": { "if": "ctx.connection.state == 'SHR'",   "field": "connection.state_description", "value": "Responder sent a SYN ACK followed by a FIN, we never saw a SYN from the originator" } },
-    { "set": { "if": "ctx.connection.state == 'OTH'",   "field": "connection.state_description", "value": "No SYN seen, just midstream traffic (a 'partial connection' that was not later closed)" } },
+    { "set": { "if": "ctx.connection?.state == 'S0'",    "field": "connection.state_description", "value": "Connection attempt seen, no reply" } },
+    { "set": { "if": "ctx.connection?.state == 'S1'",    "field": "connection.state_description", "value": "Connection established, not terminated" } },
+    { "set": { "if": "ctx.connection?.state == 'S2'",    "field": "connection.state_description", "value": "Connection established and close attempt by originator seen (but no reply from responder)" } },
+    { "set": { "if": "ctx.connection?.state == 'S3'",    "field": "connection.state_description", "value": "Connection established and close attempt by responder seen (but no reply from originator)" } },
+    { "set": { "if": "ctx.connection?.state == 'SF'",    "field": "connection.state_description", "value": "Normal SYN/FIN completion" } },
+    { "set": { "if": "ctx.connection?.state == 'REJ'",   "field": "connection.state_description", "value": "Connection attempt rejected" } },
+    { "set": { "if": "ctx.connection?.state == 'RSTO'",  "field": "connection.state_description", "value": "Connection established, originator aborted (sent a RST)" } },
+    { "set": { "if": "ctx.connection?.state == 'RSTR'",  "field": "connection.state_description", "value": "Established, responder aborted" } },
+    { "set": { "if": "ctx.connection?.state == 'RSTOS0'","field": "connection.state_description", "value": "Originator sent a SYN followed by a RST, we never saw a SYN-ACK from the responder" } },
+    { "set": { "if": "ctx.connection?.state == 'RSTRH'", "field": "connection.state_description", "value": "Responder sent a SYN ACK followed by a RST, we never saw a SYN from the (purported) originator" } },
+    { "set": { "if": "ctx.connection?.state == 'SH'",    "field": "connection.state_description", "value": "Originator sent a SYN followed by a FIN, we never saw a SYN ACK from the responder (hence the connection was 'half' open)" } },
+    { "set": { "if": "ctx.connection?.state == 'SHR'",   "field": "connection.state_description", "value": "Responder sent a SYN ACK followed by a FIN, we never saw a SYN from the originator" } },
+    { "set": { "if": "ctx.connection?.state == 'OTH'",   "field": "connection.state_description", "value": "No SYN seen, just midstream traffic (a 'partial connection' that was not later closed)" } },
     { "pipeline": 	{ "name": "zeek.common" } }
   ]
 }

salt/elasticsearch/files/ingest/zeek.cotp

@@ -0,0 +1,10 @@
+{
+  "description" : "zeek.cotp",
+  "processors" : [
+    { "remove":         { "field": ["host"],     "ignore_failure": true                                                                  } },
+    { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
+    { "rename": 	{ "field": "message2.pdu_code", 		"target_field": "cotp.pdu.code",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.pdu_name", 	"target_field": "cotp.pdu.name",		"ignore_missing": true 	} },
+    { "pipeline":       { "name": "zeek.common"                                                                                   } }
+  ]
+}

salt/elasticsearch/files/ingest/zeek.dnp3_control

@@ -0,0 +1,16 @@
+{
+  "description" : "zeek.dnp3_control",
+  "processors" : [
+    { "remove":         { "field": ["host"],     "ignore_failure": true                                                                  } },
+    { "json":		{ "field": "message",			"target_field": "message2",		        "ignore_failure": true	} },
+    { "rename":         { "field": "message2.block_type",       "target_field": "dnp3.block_type",              "ignore_missing": true  } },
+    { "rename": 	{ "field": "message2.function_code",	"target_field": "dnp3.function_code",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.index_number", 	"target_field": "dnp3.index_number",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.trip_control_code","target_field": "dnp3.trip_control_code",	"ignore_missing": true 	} },
+    { "rename":  	{ "field": "message2.operation_type",	"target_field": "dnp3.operation_type",  	"ignore_missing": true  } },
+    { "rename": 	{ "field": "message2.execute_count", 	"target_field": "dnp3.execute_count",  		"ignore_missing": true  } },
+    { "rename":         { "field": "message2.on_time",          "target_field": "dnp3.on_time",                 "ignore_missing": true  } },
+    { "rename":         { "field": "message2.off_time",         "target_field": "dnp3.off_time",                "ignore_missing": true  } },
+    { "pipeline":       { "name": "zeek.common"                                                                                   } }
+  ]
+}

salt/elasticsearch/files/ingest/zeek.dnp3_objects

@@ -0,0 +1,13 @@
+{
+  "description" : "zeek.dnp3_objects",
+  "processors" : [
+    { "remove":         { "field": ["host"],     "ignore_failure": true                                                                  } },
+    { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
+    { "rename": 	{ "field": "message2.function_code",	"target_field": "dnp3.function_code",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.object_type", 	"target_field": "dnp3.object_type",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.object_count", 	"target_field": "dnp3.object_count",		"ignore_missing": true 	} },
+    { "rename":  	{ "field": "message2.range_low",	"target_field": "dnp3.range_low",  		"ignore_missing": true  } },
+    { "rename": 	{ "field": "message2.range_high", 	"target_field": "dnp3.range_high",  		"ignore_missing": true  } },
+    { "pipeline":       { "name": "zeek.common"                                                                                   } }
+  ]
+}

salt/elasticsearch/files/ingest/zeek.ecat_aoe_info

@@ -0,0 +1,17 @@
+{
+  "description" : "zeek.ecat_aoe_info",
+  "processors" : [
+    { "remove":         { "field": ["host"],     "ignore_failure": true                                                                  } },
+    { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
+    { "rename": 	{ "field": "message2.targetid", 		"target_field": "destination.mac",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.targetport", 	"target_field": "destination.port",		"ignore_missing": true 	} },
+    { "convert":        { "field": "destination.port",       "type": "integer",                     "ignore_missing": true  } },
+    { "rename": 	{ "field": "message2.senderid", 		"target_field": "source.mac",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.senderport", 	"target_field": "source.port",		"ignore_missing": true 	} },
+    { "convert":        { "field": "source.port",       "type": "integer",                     "ignore_missing": true  } },
+    { "rename": 	{ "field": "message2.cmd", 		"target_field": "ecat.command",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.stateflags", 	"target_field": "ecat.state.flags",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.data", 		"target_field": "ecat.data",		"ignore_missing": true 	} },
+    { "pipeline":       { "name": "zeek.common"                                                                                   } }
+  ]
+}

salt/elasticsearch/files/ingest/zeek.ecat_arp_info

@@ -0,0 +1,15 @@
+{
+  "description" : "zeek.ecat_arp_info",
+  "processors" : [
+    { "remove":         { "field": ["host"],     "ignore_failure": true                                                                  } },
+    { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
+    { "rename": 	{ "field": "message2.arp_type", 		"target_field": "ecat.arp.type",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.mac_src", 	"target_field": "source.mac",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.mac_dst", 		"target_field": "destination.mac",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.SPA", 	"target_field": "source.ip",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.SHA", 		"target_field": "ecat.sender.hardware.address",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.TPA", 	"target_field": "destination.ip",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.THA", 		"target_field": "ecat.target.hardware.address",		"ignore_missing": true 	} },
+    { "pipeline":       { "name": "zeek.common"                                                                                   } }
+  ]
+}

salt/elasticsearch/files/ingest/zeek.ecat_coe_info

@@ -0,0 +1,14 @@
+{
+  "description" : "zeek.ecat_coe_info",
+  "processors" : [
+    { "remove":         { "field": ["host"],     "ignore_failure": true                                                                  } },
+    { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
+    { "rename": 	{ "field": "message2.number", 		"target_field": "ecat.message.number",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.Type", 	"target_field": "ecat.message.type",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.req_resp", 		"target_field": "ecat.request.response_type",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.index", 	"target_field": "ecat.index",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.subindex", 		"target_field": "ecat.sub.index",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.dataoffset", 	"target_field": "ecat.data_offset",		"ignore_missing": true 	} },
+    { "pipeline":       { "name": "zeek.common"                                                                                   } }
+  ]
+}

salt/elasticsearch/files/ingest/zeek.ecat_dev_info

@@ -0,0 +1,18 @@
+{
+  "description" : "zeek.ecat_dev_info",
+  "processors" : [
+    { "remove":         { "field": ["host"],     "ignore_failure": true                                                                  } },
+    { "json":		{ "field": "message",			"target_field": "message2",		          "ignore_failure": true	} },
+    { "rename": 	{ "field": "message2.slave_id",         "target_field": "ecat.slave.address",		  "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.revision", 	"target_field": "ecat.revision",		  "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.dev_type",         "target_field": "ecat.device.type",		  "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.build", 	        "target_field": "ecat.build.version",		  "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.fmmucnt", 		"target_field": "ecat.fieldbus.memory_mgmt_unit", "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.smcount", 	        "target_field": "ecat.sync.manager_count",	  "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.ports", 		"target_field": "ecat.port",		          "ignore_missing": true 	} },
+    { "convert":        { "field": "ecat.port",                 "type": "integer",                                "ignore_missing": true  } },
+    { "rename": 	{ "field": "message2.dpram", 	        "target_field": "ecat.ram.size",		  "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.features", 	"target_field": "ecat.features",		"ignore_missing": true 	} },
+    { "pipeline":       { "name": "zeek.common"                                                                                   } }
+  ]
+}

salt/elasticsearch/files/ingest/zeek.ecat_foe_info

@@ -0,0 +1,14 @@
+{
+  "description" : "zeek.ecat_foe_info",
+  "processors" : [
+    { "remove":         { "field": ["host"],     "ignore_failure": true                                                                  } },
+    { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
+    { "rename": 	{ "field": "message2.opcode", 		"target_field": "ecat.operation.code",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.reserved", 	"target_field": "ecat.reserved",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.packet_num", 		"target_field": "ecat.packet_number",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.error_code", 	"target_field": "ecat.error_code",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.filename", 		"target_field": "ecat.filename",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.data", 		"target_field": "ecat.data",		"ignore_missing": true 	} },
+    { "pipeline":       { "name": "zeek.common"                                                                                   } }
+  ]
+}

salt/elasticsearch/files/ingest/zeek.ecat_log_address

@@ -0,0 +1,14 @@
+{
+  "description" : "zeek.ecat_log_address",
+  "processors" : [
+    { "remove":         { "field": ["host"],     "ignore_failure": true                                                                  } },
+    { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
+    { "rename": 	{ "field": "message2.srcmac", 		"target_field": "source.mac",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.dstmac", 	"target_field": "destination.mac",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.Log_Addr", 	"target_field": "ecat.log.address",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.Length", 		"target_field": "ecat.length",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.Command", 		"target_field": "ecat.command",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.data", 		"target_field": "ecat.data",		"ignore_missing": true 	} },
+    { "pipeline":       { "name": "zeek.common"                                                                                   } }
+  ]
+}

salt/elasticsearch/files/ingest/zeek.ecat_registers

@@ -0,0 +1,15 @@
+{
+  "description" : "zeek.ecat_registers",
+  "processors" : [
+    { "remove":         { "field": ["host"],     "ignore_failure": true                                                                  } },
+    { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
+    { "rename": 	{ "field": "message2.srcmac", 		"target_field": "source.mac",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.dstmac", 	"target_field": "destination.mac",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.Command", 		"target_field": "ecat.command",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.Slave_Addr", 	"target_field": "ecat.slave.address",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.Register_Type", 		"target_field": "ecat.register.type",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.Register_Addr", 	"target_field": "ecat.register.address",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.data", 		"target_field": "ecat.data",		"ignore_missing": true 	} },
+    { "pipeline":       { "name": "zeek.common"                                                                                   } }
+  ]
+}

salt/elasticsearch/files/ingest/zeek.ecat_soe_info

@@ -0,0 +1,14 @@
+{
+  "description" : "zeek.ecat_soe_info",
+  "processors" : [
+    { "remove":         { "field": ["host"],     "ignore_failure": true                                                                  } },
+    { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
+    { "rename": 	{ "field": "message2.opcode", 		"target_field": "ecat.operation.code",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.incomplete", 	"target_field": "ecat.function.check",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.error", 		"target_field": "ecat.error",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.drive_num", 	"target_field": "ecat.drive.number",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.element_flags", 		"target_field": "ecat.element.flags",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.index", 	"target_field": "ecat.index",		"ignore_missing": true 	} },
+    { "pipeline":       { "name": "zeek.common"                                                                                   } }
+  ]
+}

salt/elasticsearch/files/ingest/zeek.enip

@@ -0,0 +1,16 @@
+{
+  "description" : "zeek.enip",
+  "processors" : [
+    { "remove":         { "field": ["host"],     "ignore_failure": true                                                                  } },
+    { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
+    { "rename": 	{ "field": "message2.is_orig", 		"target_field": "enip.is_orig",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.enip_command_code", 	"target_field": "enip.command_code",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.enip_command", 		"target_field": "enip.command",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.length", 	"target_field": "enip.length",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.session_handle", 		"target_field": "enip.session.handle",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.enip_status", 	"target_field": "enip.status_code",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.sender_context", 		"target_field": "enip.sender.context",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.options", 		"target_field": "enip.options",		"ignore_missing": true 	} },
+    { "pipeline":       { "name": "zeek.common"                                                                                   } }
+  ]
+}

salt/elasticsearch/files/ingest/zeek.modbus_detailed

@@ -0,0 +1,14 @@
+{
+  "description" : "zeek.modbus_detailed",
+  "processors" : [
+    { "remove":         { "field": ["host"],     "ignore_failure": true                                                                  } },
+    { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
+    { "rename": 	{ "field": "message2.unit_id", 		"target_field": "modbus.unit_id",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.func", 		"target_field": "modbus.function",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.network_direction", "target_field": "modbus.network.direction",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.address",	        "target_field": "modbus.address",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.quality", 	        "target_field": "modbus.quality",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.values", 	"target_field": "modbus.values",		"ignore_missing": true 	} },
+    { "pipeline":       { "name": "zeek.common"                                                                                   } }
+  ]
+}