Merge pull request #10193 from Security-Onion-Solutions/2.4/dev

2.4.1

.github/.gitleaks.toml

@@ -0,0 +1,546 @@
+title = "gitleaks config"
+
+# Gitleaks rules are defined by regular expressions and entropy ranges.
+# Some secrets have unique signatures which make detecting those secrets easy.
+# Examples of those secrets would be GitLab Personal Access Tokens, AWS keys, and GitHub Access Tokens.
+# All these examples have defined prefixes like `glpat`, `AKIA`, `ghp_`, etc.
+#
+# Other secrets might just be a hash which means we need to write more complex rules to verify
+# that what we are matching is a secret.
+#
+# Here is an example of a semi-generic secret
+#
+#   discord_client_secret = "8dyfuiRyq=vVc3RRr_edRk-fK__JItpZ"
+#
+# We can write a regular expression to capture the variable name (identifier),
+# the assignment symbol (like '=' or ':='), and finally the actual secret.
+# The structure of a rule to match this example secret is below:
+#
+#                                                           Beginning string
+#                                                               quotation
+#                                                                   │            End string quotation
+#                                                                   │                      │
+#                                                                   ▼                      ▼
+#    (?i)(discord[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9=_\-]{32})['\"]
+#
+#                   ▲                              ▲                                ▲
+#                   │                              │                                │
+#                   │                              │                                │
+#              identifier                  assignment symbol
+#                                                                                Secret
+#
+[[rules]]
+id = "gitlab-pat"
+description = "GitLab Personal Access Token"
+regex = '''glpat-[0-9a-zA-Z\-\_]{20}'''
+
+[[rules]]
+id = "aws-access-token"
+description = "AWS"
+regex = '''(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}'''
+
+# Cryptographic keys
+[[rules]]
+id = "PKCS8-PK"
+description = "PKCS8 private key"
+regex = '''-----BEGIN PRIVATE KEY-----'''
+
+[[rules]]
+id = "RSA-PK"
+description = "RSA private key"
+regex = '''-----BEGIN RSA PRIVATE KEY-----'''
+
+[[rules]]
+id = "OPENSSH-PK"
+description = "SSH private key"
+regex = '''-----BEGIN OPENSSH PRIVATE KEY-----'''
+
+[[rules]]
+id = "PGP-PK"
+description = "PGP private key"
+regex = '''-----BEGIN PGP PRIVATE KEY BLOCK-----'''
+
+[[rules]]
+id = "github-pat"
+description = "GitHub Personal Access Token"
+regex = '''ghp_[0-9a-zA-Z]{36}'''
+
+[[rules]]
+id = "github-oauth"
+description = "GitHub OAuth Access Token"
+regex = '''gho_[0-9a-zA-Z]{36}'''
+
+[[rules]]
+id = "SSH-DSA-PK"
+description = "SSH (DSA) private key"
+regex = '''-----BEGIN DSA PRIVATE KEY-----'''
+
+[[rules]]
+id = "SSH-EC-PK"
+description = "SSH (EC) private key"
+regex = '''-----BEGIN EC PRIVATE KEY-----'''
+
+
+[[rules]]
+id = "github-app-token"
+description = "GitHub App Token"
+regex = '''(ghu|ghs)_[0-9a-zA-Z]{36}'''
+
+[[rules]]
+id = "github-refresh-token"
+description = "GitHub Refresh Token"
+regex = '''ghr_[0-9a-zA-Z]{76}'''
+
+[[rules]]
+id = "shopify-shared-secret"
+description = "Shopify shared secret"
+regex = '''shpss_[a-fA-F0-9]{32}'''
+
+[[rules]]
+id = "shopify-access-token"
+description = "Shopify access token"
+regex = '''shpat_[a-fA-F0-9]{32}'''
+
+[[rules]]
+id = "shopify-custom-access-token"
+description = "Shopify custom app access token"
+regex = '''shpca_[a-fA-F0-9]{32}'''
+
+[[rules]]
+id = "shopify-private-app-access-token"
+description = "Shopify private app access token"
+regex = '''shppa_[a-fA-F0-9]{32}'''
+
+[[rules]]
+id = "slack-access-token"
+description = "Slack token"
+regex = '''xox[baprs]-([0-9a-zA-Z]{10,48})?'''
+
+[[rules]]
+id = "stripe-access-token"
+description = "Stripe"
+regex = '''(?i)(sk|pk)_(test|live)_[0-9a-z]{10,32}'''
+
+[[rules]]
+id = "pypi-upload-token"
+description = "PyPI upload token"
+regex = '''pypi-AgEIcHlwaS5vcmc[A-Za-z0-9\-_]{50,1000}'''
+
+[[rules]]
+id = "gcp-service-account"
+description = "Google (GCP) Service-account"
+regex = '''\"type\": \"service_account\"'''
+
+[[rules]]
+id = "heroku-api-key"
+description = "Heroku API Key"
+regex = ''' (?i)(heroku[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([0-9A-F]{8}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{12})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "slack-web-hook"
+description = "Slack Webhook"
+regex = '''https://hooks.slack.com/services/T[a-zA-Z0-9_]{8}/B[a-zA-Z0-9_]{8,12}/[a-zA-Z0-9_]{24}'''
+
+[[rules]]
+id = "twilio-api-key"
+description = "Twilio API Key"
+regex = '''SK[0-9a-fA-F]{32}'''
+
+[[rules]]
+id = "age-secret-key"
+description = "Age secret key"
+regex = '''AGE-SECRET-KEY-1[QPZRY9X8GF2TVDW0S3JN54KHCE6MUA7L]{58}'''
+
+[[rules]]
+id = "facebook-token"
+description = "Facebook token"
+regex = '''(?i)(facebook[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-f0-9]{32})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "twitter-token"
+description = "Twitter token"
+regex = '''(?i)(twitter[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-f0-9]{35,44})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "adobe-client-id"
+description = "Adobe Client ID (Oauth Web)"
+regex = '''(?i)(adobe[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-f0-9]{32})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "adobe-client-secret"
+description = "Adobe Client Secret"
+regex = '''(p8e-)(?i)[a-z0-9]{32}'''
+
+[[rules]]
+id = "alibaba-access-key-id"
+description = "Alibaba AccessKey ID"
+regex = '''(LTAI)(?i)[a-z0-9]{20}'''
+
+[[rules]]
+id = "alibaba-secret-key"
+description = "Alibaba Secret Key"
+regex = '''(?i)(alibaba[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{30})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "asana-client-id"
+description = "Asana Client ID"
+regex = '''(?i)(asana[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([0-9]{16})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "asana-client-secret"
+description = "Asana Client Secret"
+regex = '''(?i)(asana[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{32})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "atlassian-api-token"
+description = "Atlassian API token"
+regex = '''(?i)(atlassian[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{24})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "bitbucket-client-id"
+description = "Bitbucket client ID"
+regex = '''(?i)(bitbucket[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{32})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "bitbucket-client-secret"
+description = "Bitbucket client secret"
+regex = '''(?i)(bitbucket[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9_\-]{64})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "beamer-api-token"
+description = "Beamer API token"
+regex = '''(?i)(beamer[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](b_[a-z0-9=_\-]{44})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "clojars-api-token"
+description = "Clojars API token"
+regex = '''(CLOJARS_)(?i)[a-z0-9]{60}'''
+
+[[rules]]
+id = "contentful-delivery-api-token"
+description = "Contentful delivery API token"
+regex = '''(?i)(contentful[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9\-=_]{43})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "databricks-api-token"
+description = "Databricks API token"
+regex = '''dapi[a-h0-9]{32}'''
+
+[[rules]]
+id = "discord-api-token"
+description = "Discord API key"
+regex = '''(?i)(discord[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-h0-9]{64})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "discord-client-id"
+description = "Discord client ID"
+regex = '''(?i)(discord[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([0-9]{18})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "discord-client-secret"
+description = "Discord client secret"
+regex = '''(?i)(discord[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9=_\-]{32})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "doppler-api-token"
+description = "Doppler API token"
+regex = '''['\"](dp\.pt\.)(?i)[a-z0-9]{43}['\"]'''
+
+[[rules]]
+id = "dropbox-api-secret"
+description = "Dropbox API secret/key"
+regex = '''(?i)(dropbox[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{15})['\"]'''
+
+[[rules]]
+id = "dropbox--api-key"
+description = "Dropbox API secret/key"
+regex = '''(?i)(dropbox[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{15})['\"]'''
+
+[[rules]]
+id = "dropbox-short-lived-api-token"
+description = "Dropbox short lived API token"
+regex = '''(?i)(dropbox[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](sl\.[a-z0-9\-=_]{135})['\"]'''
+
+[[rules]]
+id = "dropbox-long-lived-api-token"
+description = "Dropbox long lived API token"
+regex = '''(?i)(dropbox[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"][a-z0-9]{11}(AAAAAAAAAA)[a-z0-9\-_=]{43}['\"]'''
+
+[[rules]]
+id = "duffel-api-token"
+description = "Duffel API token"
+regex = '''['\"]duffel_(test|live)_(?i)[a-z0-9_-]{43}['\"]'''
+
+[[rules]]
+id = "dynatrace-api-token"
+description = "Dynatrace API token"
+regex = '''['\"]dt0c01\.(?i)[a-z0-9]{24}\.[a-z0-9]{64}['\"]'''
+
+[[rules]]
+id = "easypost-api-token"
+description = "EasyPost API token"
+regex = '''['\"]EZAK(?i)[a-z0-9]{54}['\"]'''
+
+[[rules]]
+id = "easypost-test-api-token"
+description = "EasyPost test API token"
+regex = '''['\"]EZTK(?i)[a-z0-9]{54}['\"]'''
+
+[[rules]]
+id = "fastly-api-token"
+description = "Fastly API token"
+regex = '''(?i)(fastly[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9\-=_]{32})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "finicity-client-secret"
+description = "Finicity client secret"
+regex = '''(?i)(finicity[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{20})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "finicity-api-token"
+description = "Finicity API token"
+regex = '''(?i)(finicity[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-f0-9]{32})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "flutterwave-public-key"
+description = "Flutterwave public key"
+regex = '''FLWPUBK_TEST-(?i)[a-h0-9]{32}-X'''
+
+[[rules]]
+id = "flutterwave-secret-key"
+description = "Flutterwave secret key"
+regex = '''FLWSECK_TEST-(?i)[a-h0-9]{32}-X'''
+
+[[rules]]
+id = "flutterwave-enc-key"
+description = "Flutterwave encrypted key"
+regex = '''FLWSECK_TEST[a-h0-9]{12}'''
+
+[[rules]]
+id = "frameio-api-token"
+description = "Frame.io API token"
+regex = '''fio-u-(?i)[a-z0-9\-_=]{64}'''
+
+[[rules]]
+id = "gocardless-api-token"
+description = "GoCardless API token"
+regex = '''['\"]live_(?i)[a-z0-9\-_=]{40}['\"]'''
+
+[[rules]]
+id = "grafana-api-token"
+description = "Grafana API token"
+regex = '''['\"]eyJrIjoi(?i)[a-z0-9\-_=]{72,92}['\"]'''
+
+[[rules]]
+id = "hashicorp-tf-api-token"
+description = "HashiCorp Terraform user/org API token"
+regex = '''['\"](?i)[a-z0-9]{14}\.atlasv1\.[a-z0-9\-_=]{60,70}['\"]'''
+
+[[rules]]
+id = "hubspot-api-token"
+description = "HubSpot API token"
+regex = '''(?i)(hubspot[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-h0-9]{8}-[a-h0-9]{4}-[a-h0-9]{4}-[a-h0-9]{4}-[a-h0-9]{12})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "intercom-api-token"
+description = "Intercom API token"
+regex = '''(?i)(intercom[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9=_]{60})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "intercom-client-secret"
+description = "Intercom client secret/ID"
+regex = '''(?i)(intercom[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-h0-9]{8}-[a-h0-9]{4}-[a-h0-9]{4}-[a-h0-9]{4}-[a-h0-9]{12})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "ionic-api-token"
+description = "Ionic API token"
+regex = '''(?i)(ionic[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](ion_[a-z0-9]{42})['\"]'''
+
+[[rules]]
+id = "linear-api-token"
+description = "Linear API token"
+regex = '''lin_api_(?i)[a-z0-9]{40}'''
+
+[[rules]]
+id = "linear-client-secret"
+description = "Linear client secret/ID"
+regex = '''(?i)(linear[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-f0-9]{32})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "lob-api-key"
+description = "Lob API Key"
+regex = '''(?i)(lob[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]((live|test)_[a-f0-9]{35})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "lob-pub-api-key"
+description = "Lob Publishable API Key"
+regex = '''(?i)(lob[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]((test|live)_pub_[a-f0-9]{31})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "mailchimp-api-key"
+description = "Mailchimp API key"
+regex = '''(?i)(mailchimp[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-f0-9]{32}-us20)['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "mailgun-private-api-token"
+description = "Mailgun private API token"
+regex = '''(?i)(mailgun[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](key-[a-f0-9]{32})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "mailgun-pub-key"
+description = "Mailgun public validation key"
+regex = '''(?i)(mailgun[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](pubkey-[a-f0-9]{32})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "mailgun-signing-key"
+description = "Mailgun webhook signing key"
+regex = '''(?i)(mailgun[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-h0-9]{32}-[a-h0-9]{8}-[a-h0-9]{8})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "mapbox-api-token"
+description = "Mapbox API token"
+regex = '''(?i)(pk\.[a-z0-9]{60}\.[a-z0-9]{22})'''
+
+[[rules]]
+id = "messagebird-api-token"
+description = "MessageBird API token"
+regex = '''(?i)(messagebird[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{25})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "messagebird-client-id"
+description = "MessageBird API client ID"
+regex = '''(?i)(messagebird[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-h0-9]{8}-[a-h0-9]{4}-[a-h0-9]{4}-[a-h0-9]{4}-[a-h0-9]{12})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "new-relic-user-api-key"
+description = "New Relic user API Key"
+regex = '''['\"](NRAK-[A-Z0-9]{27})['\"]'''
+
+[[rules]]
+id = "new-relic-user-api-id"
+description = "New Relic user API ID"
+regex = '''(?i)(newrelic[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([A-Z0-9]{64})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "new-relic-browser-api-token"
+description = "New Relic ingest browser API token"
+regex = '''['\"](NRJS-[a-f0-9]{19})['\"]'''
+
+[[rules]]
+id = "npm-access-token"
+description = "npm access token"
+regex = '''['\"](npm_(?i)[a-z0-9]{36})['\"]'''
+
+[[rules]]
+id = "planetscale-password"
+description = "PlanetScale password"
+regex = '''pscale_pw_(?i)[a-z0-9\-_\.]{43}'''
+
+[[rules]]
+id = "planetscale-api-token"
+description = "PlanetScale API token"
+regex = '''pscale_tkn_(?i)[a-z0-9\-_\.]{43}'''
+
+[[rules]]
+id = "postman-api-token"
+description = "Postman API token"
+regex = '''PMAK-(?i)[a-f0-9]{24}\-[a-f0-9]{34}'''
+
+[[rules]]
+id = "pulumi-api-token"
+description = "Pulumi API token"
+regex = '''pul-[a-f0-9]{40}'''
+
+[[rules]]
+id = "rubygems-api-token"
+description = "Rubygem API token"
+regex = '''rubygems_[a-f0-9]{48}'''
+
+[[rules]]
+id = "sendgrid-api-token"
+description = "SendGrid API token"
+regex = '''SG\.(?i)[a-z0-9_\-\.]{66}'''
+
+[[rules]]
+id = "sendinblue-api-token"
+description = "Sendinblue API token"
+regex = '''xkeysib-[a-f0-9]{64}\-(?i)[a-z0-9]{16}'''
+
+[[rules]]
+id = "shippo-api-token"
+description = "Shippo API token"
+regex = '''shippo_(live|test)_[a-f0-9]{40}'''
+
+[[rules]]
+id = "linkedin-client-secret"
+description = "LinkedIn Client secret"
+regex = '''(?i)(linkedin[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z]{16})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "linkedin-client-id"
+description = "LinkedIn Client ID"
+regex = '''(?i)(linkedin[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{14})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "twitch-api-token"
+description = "Twitch API token"
+regex = '''(?i)(twitch[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{30})['\"]'''
+secretGroup = 3
+
+[[rules]]
+id = "typeform-api-token"
+description = "Typeform API token"
+regex = '''(?i)(typeform[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}(tfp_[a-z0-9\-_\.=]{59})'''
+secretGroup = 3
+
+[[rules]]
+id = "generic-api-key"
+description = "Generic API Key"
+regex = '''(?i)((key|api[^Version]|token|secret|password)[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([0-9a-zA-Z\-_=]{8,64})['\"]'''
+entropy = 3.7
+secretGroup = 4
+
+
+[allowlist]
+description = "global allow lists"
+regexes = ['''219-09-9999''', '''078-05-1120''', '''(9[0-9]{2}|666)-\d{2}-\d{4}''', '''RPM-GPG-KEY.*''']
+paths = [
+    '''gitleaks.toml''',
+    '''(.*?)(jpg|gif|doc|pdf|bin|svg|socket)$''',
+    '''(go.mod|go.sum)$''',
+    
+    '''salt/nginx/files/enterprise-attack.json'''
+]

.github/workflows/contrib.yml

@@ -0,0 +1,24 @@
+name: contrib
+on:
+  issue_comment:
+    types: [created]
+  pull_request_target:
+    types: [opened,closed,synchronize]
+
+jobs:
+  CLAssistant:
+    runs-on: ubuntu-latest
+    steps:
+      - name: "Contributor Check"
+        if: (github.event.comment.body == 'recheck' || github.event.comment.body == 'I have read the CLA Document and I hereby sign the CLA') || github.event_name == 'pull_request_target'
+        uses: cla-assistant/github-action@v2.1.3-beta
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          PERSONAL_ACCESS_TOKEN : ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+        with:
+          path-to-signatures: 'signatures_v1.json'
+          path-to-document: 'https://securityonionsolutions.com/cla' 
+          allowlist: dependabot[bot],jertel,dougburks,TOoSmOotH,weslambert,defensivedepth,m0duspwnens
+          remote-organization-name: Security-Onion-Solutions
+          remote-repository-name: licensing
+

.github/workflows/leaktest.yml

@@ -1,6 +1,6 @@
 name: leak-test
 
-on: [push,pull_request]
+on: [pull_request]
 
 jobs:
   build:
@@ -12,4 +12,6 @@ jobs:
         fetch-depth: '0'
 
     - name: Gitleaks
-      uses: zricethezav/gitleaks-action@master
+      uses: gitleaks/gitleaks-action@v1.6.0
+      with:
+        config-path: .github/.gitleaks.toml

.github/workflows/pythontest.yml

@@ -0,0 +1,37 @@
+name: python-test
+
+on:
+  push:
+    paths: 
+      - "salt/sensoroni/files/analyzers/**"
+  pull_request:
+    paths:
+      - "salt/sensoroni/files/analyzers/**"
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10"]
+        python-code-path: ["salt/sensoroni/files/analyzers"]
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install flake8 pytest pytest-cov
+        find . -name requirements.txt -exec pip install -r {} \;
+    - name: Lint with flake8
+      run: |
+        flake8 ${{ matrix.python-code-path }} --show-source --max-complexity=12  --doctests --max-line-length=200 --statistics
+    - name: Test with pytest
+      run: |
+        pytest ${{ matrix.python-code-path }} --cov=${{ matrix.python-code-path }} --doctest-modules --cov-report=term --cov-fail-under=100 --cov-config=${{ matrix.python-code-path }}/pytest.ini

.gitignore

@@ -56,4 +56,15 @@ $RECYCLE.BIN/
 # Windows shortcuts
 *.lnk
 
-# End of https://www.gitignore.io/api/macos,windows
+# End of https://www.gitignore.io/api/macos,windows
+
+# Pytest output
+__pycache__
+.pytest_cache
+.coverage
+*.pyc
+.venv
+
+# Analyzer dev/test config files
+*_dev.yaml
+site-packages

CONTRIBUTING.md

@@ -0,0 +1,46 @@
+# Contributing to Security Onion
+
+### Questions, suggestions, and general comments
+* Security Onion uses GitHub's [Discussions](https://github.com/Security-Onion-Solutions/securityonion/discussions) to provide a forum where the community and developers can interact as well as ask and answer questions.
+
+### Reporting a bug
+* The primary place to report unexpected behavior or possible bugs is the repo's [Discussions forum](https://github.com/Security-Onion-Solutions/securityonion/discussions).
+
+*  **If you are familiar with the current version of Security Onion and are confident you've discovered a bug**, first ensure there is not already an issue present by searching the open [issues](https://github.com/Security-Onion-Solutions/securityonion/issues). If there is, a thumbs up :+1: is a great way to show this bug is affecting you too.
+
+* If an issue doesn't exist, [open a new one](https://github.com/Security-Onion-Solutions/securityonion/issues/new), following the directions in the issue template. This means including:
+  * **System information** and how Security Onion was installed
+  * **Log files** relevant to the bug report
+  * **Reproduction steps** 
+
+### Contributing code
+
+* **All commits must be signed** with a valid key that has been added to your GitHub account. Each commit should have the "**Verified**" tag when viewed on GitHub as shown below:
+  
+  <img src="./assets/images/verified-commit-1.png" width="450">
+
+* If an issue does not already exist for the bug or feature for which you are submitting a pull request, [create one](https://github.com/Security-Onion-Solutions/securityonion/issues/new) with the relevant prefix. (**`FIX:`** for bug fixes, **`FEATURE:`** for new features.)
+
+* Link the PR to the related issue, either using [keywords](https://docs.github.com/en/issues/tracking-your-work-with-issues/creating-issues/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword) in the PR description, or [manually](https://docs.github.com/en/issues/tracking-your-work-with-issues/creating-issues/linking-a-pull-request-to-an-issue#manually-linking-a-pull-request-to-an-issue).
+
+* **Pull requests should be opened against the `dev` branch of this repo**, and should clearly describe the problem and solution.
+
+* Be sure you have tested your changes and are confident they will not break other parts of the product.
+
+* See this document's [code styling and conventions section](#code-style-and-conventions) below to be sure your PR fits our code requirements prior to submitting.
+
+* Change behavior (fix a bug, add a new feature) separately from refactoring code. Refactor pull requests are welcome, but ensure your new code behaves exactly the same as the old.
+
+* **Do not refactor code for non-functional reasons**. If you are submitting a pull request that refactors code, ensure the refactor is improving the functionality of the code you're refactoring (e.g. decreasing complexity, removing reliance on 3rd party tools, improving performance).
+
+* Before submitting a PR with significant changes to the project, [start a discussion](https://github.com/Security-Onion-Solutions/securityonion/discussions/new) explaining what you hope to acheive. The project maintainers will provide feedback and determine whether your goal aligns with the project. 
+
+
+### Code style and conventions
+* **Keep code [DRY](https://en.wikipedia.org/wiki/Don%27t_repeat_yourself)**. For example, Bash code used by multiple scripts will likely best be added to <span style="white-space: nowrap;">[`so-common`](salt/common/tools/sbin/so-common)</span>.
+
+* All new Bash code should pass [ShellCheck](https://www.shellcheck.net/) analysis. Where errors can be *safely* [ignored](https://github.com/koalaman/shellcheck/wiki/Ignore), the relevant disable directive should be accompanied by a brief explanation as to why the error is being ignored.
+
+* **Ensure all YAML (this includes Salt states and pillars) is properly formatted**. The spec for YAML v1.2 can be found [here](https://yaml.org/spec/1.2/spec.html), however there are numerous online resources with simpler descriptions of its formatting rules. 
+
+* **All code of any language should match the style of other code of that same language within the project.** Be sure that any changes you make do not break from the pre-existing style of Security Onion code.

HOTFIX

@@ -0,0 +1 @@
+

README.md

@@ -1,35 +1,41 @@
-## Security Onion 2.3.20
+## Security Onion 2.4 Beta 2
 
-Security Onion 2.3.20 is here!
+Security Onion 2.4 Beta 2 is here!
 
 ## Screenshots
 
 Alerts
-![Alerts](https://raw.githubusercontent.com/security-onion-solutions/securityonion/master/screenshots/alerts-1.png)
+![Alerts](./assets/images/screenshots/alerts.png)
+
+Dashboards
+![Dashboards](./assets/images/screenshots/dashboards.png)
 
 Hunt
-![Hunt](https://raw.githubusercontent.com/security-onion-solutions/securityonion/master/screenshots/hunt-1.png)
+![Hunt](./assets/images/screenshots/hunt.png)
+
+Cases
+![Cases](./assets/images/screenshots/cases-comments.png)
 
 ### Release Notes
 
-https://docs.securityonion.net/en/2.3/release-notes.html
+https://docs.securityonion.net/en/2.4/release-notes.html
 
 ### Requirements
 
-https://docs.securityonion.net/en/2.3/hardware.html
+https://docs.securityonion.net/en/2.4/hardware.html
 
 ### Download
 
-https://docs.securityonion.net/en/2.3/download.html
+https://docs.securityonion.net/en/2.4/download.html
 
 ### Installation
 
-https://docs.securityonion.net/en/2.3/installation.html
+https://docs.securityonion.net/en/2.4/installation.html
 
 ### FAQ
 
-https://docs.securityonion.net/en/2.3/faq.html
+https://docs.securityonion.net/en/2.4/faq.html
 
 ### Feedback
 
-https://docs.securityonion.net/en/2.3/community-support.html
+https://docs.securityonion.net/en/2.4/community-support.html

SECURITY.md

@@ -0,0 +1,22 @@
+# Security Policy
+
+## Supported Versions
+
+| Version | Supported          |
+| ------- | ------------------ |
+| 2.4.x   | :white_check_mark: |
+| 2.3.x   | :white_check_mark: |
+| 16.04.x | :x:                |
+
+Security Onion 16.04 has reached End Of Life and is no longer supported.
+
+## Reporting a Vulnerability
+
+If you have any security concerns regarding Security Onion or believe you have uncovered a vulnerability, please follow these steps:
+
+- send an email to security@securityonion.net
+- include a description of the issue and steps to reproduce
+- please use plain text format (no Word documents or PDF files)
+- please do not disclose publicly until we have had sufficient time to resolve the issue
+
+This security address should be used only for undisclosed vulnerabilities. Dealing with fixed issues or general questions on how to use Security Onion should be handled via the normal support channels.

VERIFY_ISO.md

@@ -1,50 +1 @@
-### 2.3.20 ISO image built on 2020/12/20
-
-### Download and Verify
-
-2.3.20 ISO image:  
-https://download.securityonion.net/file/securityonion/securityonion-2.3.20.iso
-
-MD5: E348FA65A46FD3FBA0D574D9C1A0582D  
-SHA1: 4A6E6D4E0B31ECA1B72E642E3DB2C186B59009D6  
-SHA256: 25DE77097903640771533FA13094D0720A032B70223875F8C77A92F5C44CA687 
-
-Signature for ISO image:  
-https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.20.iso.sig
-
-Signing key:  
-https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS  
-
-For example, here are the steps you can use on most Linux distributions to download and verify our Security Onion ISO image.
-
-Download and import the signing key:  
-```
-wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS -O - | gpg --import -  
-```
-
-Download the signature file for the ISO:  
-```
-wget https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.20.iso.sig
-```
-
-Download the ISO image:  
-```
-wget https://download.securityonion.net/file/securityonion/securityonion-2.3.20.iso
-```
-
-Verify the downloaded ISO image using the signature file:  
-```
-gpg --verify securityonion-2.3.20.iso.sig securityonion-2.3.20.iso
-```
-
-The output should show "Good signature" and the Primary key fingerprint should match what's shown below:
-```
-gpg: Signature made Sun 20 Dec 2020 11:11:28 AM EST using RSA key ID FE507013
-gpg: Good signature from "Security Onion Solutions, LLC <info@securityonionsolutions.com>"
-gpg: WARNING: This key is not certified with a trusted signature!
-gpg:          There is no indication that the signature belongs to the owner.
-Primary key fingerprint: C804 A93D 36BE 0C73 3EA1  9644 7C10 60B7 FE50 7013
-```
-
-Once you've verified the ISO image, you're ready to proceed to our Installation guide:  
-https://docs.securityonion.net/en/2.3/installation.html
+### An ISO will be available starting in RC1. 

VERSION

@@ -1 +1 @@
-2.3.20
+2.4.1

files/firewall/assigned_hostgroups.local.map.yaml

@@ -1,8 +1,8 @@
-{% import_yaml 'firewall/portgroups.yaml' as default_portgroups %}
-{% set default_portgroups = default_portgroups.firewall.aliases.ports %}
-{% import_yaml 'firewall/portgroups.local.yaml' as local_portgroups %}
-{% if local_portgroups.firewall.aliases.ports %}
-  {% set local_portgroups = local_portgroups.firewall.aliases.ports %}
+{% import_yaml 'firewall/ports/ports.yaml' as default_portgroups %}
+{% set default_portgroups = default_portgroups.firewall.ports %}
+{% import_yaml 'firewall/ports/ports.local.yaml' as local_portgroups %}
+{% if local_portgroups.firewall.ports %}
+  {% set local_portgroups = local_portgroups.firewall.ports %}
 {% else %}
   {% set local_portgroups = {} %}
 {% endif %}
@@ -13,9 +13,11 @@ role:
   fleet:
   heavynode:
   helixsensor:
+  idh:
   import:
   manager:
   managersearch:
+  receiver:
   standalone:
   searchnode:
-  sensor:
+  sensor:

files/firewall/hostgroups.local.yaml

@@ -1,70 +0,0 @@
-firewall:
-  hostgroups:
-    analyst:
-      ips:
-        delete:
-        insert:
-    beats_endpoint:
-      ips:
-        delete:
-        insert:
-    beats_endpoint_ssl:
-      ips:
-        delete:
-        insert:
-    elasticsearch_rest:
-      ips:
-        delete:
-        insert:
-    fleet:
-      ips:
-        delete:
-        insert:
-    heavy_node:
-      ips:
-        delete:
-        insert:
-    manager:
-      ips:
-        delete:
-        insert:
-    minion:
-      ips:
-        delete:
-        insert:
-    node:
-      ips:
-        delete:
-        insert:
-    osquery_endpoint:
-      ips:
-        delete:
-        insert:
-    search_node:
-      ips:
-        delete:
-        insert:
-    sensor:
-      ips:
-        delete:
-        insert:
-    strelka_frontend:
-      ips:
-        delete:
-        insert:
-    syslog:
-      ips:
-        delete:
-        insert:
-    wazuh_agent:
-      ips:
-        delete:
-        insert:
-    wazuh_api:
-      ips:
-        delete:
-        insert:
-    wazuh_authd:
-      ips:
-        delete:
-        insert:

files/firewall/portgroups.local.yaml

@@ -1,3 +0,0 @@
-firewall:
-  aliases:
-    ports:

files/firewall/ports/ports.local.yaml

@@ -0,0 +1,2 @@
+firewall:
+  ports:

files/salt/master/master

@@ -13,6 +13,8 @@
 # user: socore
 
 log_file: /opt/so/log/salt/master
+log_level_logfile: info
+log_level: info
 
 #####      File Server settings      #####
 ##########################################
@@ -62,6 +64,4 @@ peer:
   .*:
     - x509.sign_remote_certificate
 
-reactor:
-  - 'so/fleet':
-    - salt://reactor/fleet.sls
+

pillar/data/addtotab.sh

@@ -45,12 +45,10 @@ echo "    rootfs: $ROOTFS" >> $local_salt_dir/pillar/data/$TYPE.sls
 echo "    nsmfs: $NSM" >> $local_salt_dir/pillar/data/$TYPE.sls
 if [ $TYPE == 'sensorstab' ]; then
   echo "    monint: bond0" >> $local_salt_dir/pillar/data/$TYPE.sls
-  salt-call state.apply grafana queue=True
 fi
 if [ $TYPE == 'evaltab' ] || [ $TYPE == 'standalonetab' ]; then
   echo "    monint: bond0" >> $local_salt_dir/pillar/data/$TYPE.sls
   if [ ! $10 ]; then
-    salt-call state.apply grafana queue=True
     salt-call state.apply utility queue=True
   fi
 fi

pillar/docker/config.sls

@@ -1,208 +0,0 @@
-{%- set FLEETMANAGER = salt['pillar.get']('global:fleet_manager', False) -%}
-{%- set FLEETNODE = salt['pillar.get']('global:fleet_node', False) -%}
-{% set WAZUH = salt['pillar.get']('manager:wazuh', '0') %}
-{% set THEHIVE = salt['pillar.get']('manager:thehive', '0') %}
-{% set PLAYBOOK = salt['pillar.get']('manager:playbook', '0') %}
-{% set FREQSERVER = salt['pillar.get']('manager:freq', '0') %}
-{% set DOMAINSTATS = salt['pillar.get']('manager:domainstats', '0') %}
-{% set ZEEKVER = salt['pillar.get']('global:mdengine', 'COMMUNITY') %}
-{% set GRAFANA = salt['pillar.get']('manager:grafana', '0') %}
-
-eval:
-  containers:
-    - so-nginx
-    - so-telegraf
-    {% if  GRAFANA == '1' %}
-    - so-influxdb
-    - so-grafana
-    {% endif %}
-    - so-dockerregistry
-    - so-soc
-    - so-kratos
-    - so-idstools
-    {% if FLEETMANAGER %}
-    - so-mysql
-    - so-fleet
-    - so-redis
-    {% endif %}
-    - so-elasticsearch
-    - so-logstash
-    - so-kibana
-    - so-steno
-    - so-suricata
-    - so-zeek
-    - so-curator
-    - so-elastalert
-    {% if WAZUH != '0' %}
-    - so-wazuh
-    {% endif %}
-    - so-soctopus
-    {% if THEHIVE != '0' %}
-    - so-thehive
-    - so-thehive-es
-    - so-cortex
-    {% endif %}
-    {% if PLAYBOOK != '0' %}
-    - so-playbook
-    {% endif %}
-    {% if FREQSERVER != '0' %}
-    - so-freqserver
-    {% endif %}
-    {% if DOMAINSTATS != '0' %}
-    - so-domainstats
-    {% endif %}
-heavy_node:
-  containers:
-    - so-nginx
-    - so-telegraf
-    - so-redis
-    - so-logstash
-    - so-elasticsearch
-    - so-curator
-    - so-steno
-    - so-suricata
-    - so-wazuh
-    - so-filebeat
-    {% if ZEEKVER != 'SURICATA' %}
-    - so-zeek
-    {% endif %}
-helix:
-  containers:
-    - so-nginx
-    - so-telegraf
-    - so-idstools
-    - so-steno
-    - so-zeek
-    - so-redis
-    - so-logstash
-    - so-filebeat
-hot_node:
-  containers:
-    - so-nginx
-    - so-telegraf
-    - so-logstash
-    - so-elasticsearch
-    - so-curator
-manager_search:
-  containers:
-    - so-nginx
-    - so-telegraf
-    - so-soc
-    - so-kratos
-    - so-acng
-    - so-idstools
-    - so-redis
-    - so-logstash
-    - so-elasticsearch
-    - so-curator
-    - so-kibana
-    - so-elastalert
-    - so-filebeat
-    - so-soctopus
-    {% if FLEETMANAGER %}
-    - so-mysql
-    - so-fleet
-    - so-redis
-    {% endif %}
-    {% if WAZUH != '0' %}
-    - so-wazuh
-    {% endif %}
-    - so-soctopus
-    {% if THEHIVE != '0' %}
-    - so-thehive
-    - so-thehive-es
-    - so-cortex
-    {% endif %}
-    {% if PLAYBOOK != '0' %}
-    - so-playbook
-    {% endif %}
-    {% if FREQSERVER != '0' %}
-    - so-freqserver
-    {% endif %}
-    {% if DOMAINSTATS != '0' %}
-    - so-domainstats
-    {% endif %}
-manager:
-  containers:
-    - so-dockerregistry
-    - so-nginx
-    - so-telegraf
-    {% if  GRAFANA == '1' %}
-    - so-influxdb
-    - so-grafana
-    {% endif %}
-    - so-soc
-    - so-kratos
-    - so-acng
-    - so-idstools
-    - so-redis
-    - so-elasticsearch
-    - so-logstash
-    - so-kibana
-    - so-elastalert
-    - so-filebeat
-    {% if FLEETMANAGER %}
-    - so-mysql
-    - so-fleet
-    - so-redis
-    {% endif %}
-    {% if WAZUH != '0' %}
-    - so-wazuh
-    {% endif %}
-    - so-soctopus
-    {% if THEHIVE != '0' %}
-    - so-thehive
-    - so-thehive-es
-    - so-cortex
-    {% endif %}
-    {% if PLAYBOOK != '0' %}
-    - so-playbook
-    {% endif %}
-    {% if FREQSERVER != '0' %}
-    - so-freqserver
-    {% endif %}
-    {% if DOMAINSTATS != '0' %}
-    - so-domainstats
-    {% endif %}
-parser_node:
-  containers:
-    - so-nginx
-    - so-telegraf
-    - so-logstash
-search_node:
-  containers:
-    - so-nginx
-    - so-telegraf
-    - so-logstash
-    - so-elasticsearch
-    - so-curator
-    - so-filebeat
-    {% if WAZUH != '0' %}
-    - so-wazuh
-    {% endif %}
-sensor:
-  containers:
-    - so-nginx
-    - so-telegraf
-    - so-steno
-    - so-suricata
-    {% if ZEEKVER != 'SURICATA' %}
-    - so-zeek
-    {% endif %}
-    - so-wazuh
-    - so-filebeat
-warm_node:
-  containers:
-    - so-nginx
-    - so-telegraf
-    - so-elasticsearch
-fleet:
-  containers:
-    {% if FLEETNODE %}
-    - so-mysql
-    - so-fleet
-    - so-redis
-    - so-filebeat
-    - so-nginx
-    - so-telegraf
-    {% endif %}

pillar/elasticsearch/eval.sls

@@ -1,13 +1,2 @@
 elasticsearch:
   templates:
-    - so/so-beats-template.json.jinja
-    - so/so-common-template.json
-    - so/so-firewall-template.json.jinja
-    - so/so-flow-template.json.jinja
-    - so/so-ids-template.json.jinja
-    - so/so-import-template.json.jinja
-    - so/so-osquery-template.json.jinja
-    - so/so-ossec-template.json.jinja
-    - so/so-strelka-template.json.jinja
-    - so/so-syslog-template.json.jinja
-    - so/so-zeek-template.json.jinja

pillar/elasticsearch/index_templates.sls

@@ -0,0 +1,2 @@
+elasticsearch:
+  index_settings:

pillar/elasticsearch/manager.sls

@@ -1,13 +1,2 @@
 elasticsearch:
   templates:
-    - so/so-beats-template.json.jinja
-    - so/so-common-template.json
-    - so/so-firewall-template.json.jinja
-    - so/so-flow-template.json.jinja
-    - so/so-ids-template.json.jinja
-    - so/so-import-template.json.jinja
-    - so/so-osquery-template.json.jinja
-    - so/so-ossec-template.json.jinja
-    - so/so-strelka-template.json.jinja
-    - so/so-syslog-template.json.jinja
-    - so/so-zeek-template.json.jinja

pillar/elasticsearch/search.sls

@@ -1,13 +1,2 @@
 elasticsearch:
   templates:
-    - so/so-beats-template.json.jinja
-    - so/so-common-template.json
-    - so/so-firewall-template.json.jinja
-    - so/so-flow-template.json.jinja
-    - so/so-ids-template.json.jinja
-    - so/so-import-template.json.jinja
-    - so/so-osquery-template.json.jinja
-    - so/so-ossec-template.json.jinja
-    - so/so-strelka-template.json.jinja
-    - so/so-syslog-template.json.jinja
-    - so/so-zeek-template.json.jinja

pillar/logrotate/init.sls

@@ -8,4 +8,6 @@ logrotate:
     create
     extension .log
     dateext
-    dateyesterday
+    dateyesterday
+  group_conf: |
+    su root socore

pillar/logstash/fleet.sls

@@ -0,0 +1,6 @@
+logstash:
+  pipelines:
+    fleet:
+      config:
+        - so/0012_input_elastic_agent.conf     
+        - so/9806_output_lumberjack_fleet.conf.jinja

pillar/logstash/init.sls

@@ -1,7 +1,10 @@
 logstash:
   docker_options:
     port_bindings:
+      - 0.0.0.0:3765:3765
       - 0.0.0.0:5044:5044
+      - 0.0.0.0:5055:5055
+      - 0.0.0.0:5056:5056
       - 0.0.0.0:5644:5644
       - 0.0.0.0:6050:6050
       - 0.0.0.0:6051:6051

pillar/logstash/manager.sls

@@ -1,9 +1,8 @@
-{%- set PIPELINE = salt['pillar.get']('global:pipeline', 'redis') %}
 logstash:
   pipelines:
     manager:
       config:
-        - so/0009_input_beats.conf      
-        - so/0010_input_hhbeats.conf
-        - so/9999_output_redis.conf.jinja
-        
+        - so/0011_input_endgame.conf
+        - so/0012_input_elastic_agent.conf
+        - so/0013_input_lumberjack_fleet.conf
+        - so/9999_output_redis.conf.jinja

pillar/logstash/nodes.sls

@@ -0,0 +1,31 @@
+{% set node_types = {} %}
+{% set cached_grains = salt.saltutil.runner('cache.grains', tgt='*') %}
+{% for minionid, ip in salt.saltutil.runner(
+    'mine.get',
+    tgt='G@role:so-manager or G@role:so-managersearch or G@role:so-standalone or G@role:so-searchnode or G@role:so-heavynode or G@role:so-receiver or G@role:so-helix ',
+    fun='network.ip_addrs',
+    tgt_type='compound') | dictsort()
+%}
+
+{%   set hostname = cached_grains[minionid]['host'] %}
+{%   set node_type = minionid.split('_')[1] %}
+{%   if node_type not in node_types.keys() %}
+{%     do node_types.update({node_type: {hostname: ip[0]}}) %}
+{%   else %}
+{%     if hostname not in node_types[node_type] %}
+{%       do node_types[node_type].update({hostname: ip[0]}) %}
+{%     else %}
+{%       do node_types[node_type][hostname].update(ip[0]) %}
+{%     endif %}
+{%   endif %}
+{% endfor %}
+
+logstash:
+  nodes:
+{% for node_type, values in node_types.items() %}
+    {{node_type}}:
+{%   for hostname, ip in values.items() %}
+      {{hostname}}:
+        ip: {{ip}}
+{%   endfor %}
+{% endfor %}

pillar/logstash/receiver.sls

@@ -0,0 +1,8 @@
+logstash:
+  pipelines:
+    receiver:
+      config:
+        - so/0011_input_endgame.conf
+        - so/0012_input_elastic_agent.conf
+        - so/9999_output_redis.conf.jinja
+        

pillar/logstash/search.sls

@@ -1,14 +1,7 @@
-{%- set PIPELINE = salt['pillar.get']('global:pipeline', 'minio') %}
 logstash:
   pipelines:
     search:
       config:
         - so/0900_input_redis.conf.jinja
-        - so/9000_output_zeek.conf.jinja
-        - so/9002_output_import.conf.jinja
-        - so/9034_output_syslog.conf.jinja
-        - so/9100_output_osquery.conf.jinja
-        - so/9400_output_suricata.conf.jinja
-        - so/9500_output_beats.conf.jinja
-        - so/9600_output_ossec.conf.jinja
-        - so/9700_output_strelka.conf.jinja
+        - so/9805_output_elastic_agent.conf.jinja
+        - so/9900_output_endgame.conf.jinja

pillar/node_data/ips.sls

@@ -0,0 +1,31 @@
+{% set node_types = {} %}
+{% set manage_alived = salt.saltutil.runner('manage.alived', show_ip=True) %}
+{% for minionid, ip in salt.saltutil.runner('mine.get', tgt='*', fun='network.ip_addrs', tgt_type='glob') | dictsort() %}
+{%   set hostname = minionid.split('_')[0] %}
+{%   set node_type = minionid.split('_')[1] %}
+{%   set is_alive = False %}
+{%   if minionid in manage_alived.keys() %}
+{%     if ip[0] == manage_alived[minionid] %}
+{%       set is_alive = True %}
+{%     endif %}
+{%   endif %}
+{%   if node_type not in node_types.keys() %}
+{%     do node_types.update({node_type: {hostname: {'ip':ip[0], 'alive':is_alive }}}) %}
+{%   else %}
+{%     if hostname not in node_types[node_type] %}
+{%       do node_types[node_type].update({hostname: {'ip':ip[0], 'alive':is_alive}}) %}
+{%     else %}
+{%       do node_types[node_type][hostname].update({'ip':ip[0], 'alive':is_alive}) %}
+{%     endif %}
+{%   endif %}
+{% endfor %}
+
+node_data:
+{% for node_type, host_values in node_types.items() %}
+{%   for hostname, details in host_values.items() %}
+  {{hostname}}:
+    ip: {{details.ip}}
+    alive: {{ details.alive }}
+    role: {{node_type}}
+{%   endfor %}
+{% endfor %}

pillar/top.sls

@@ -1,92 +1,244 @@
 base:
   '*':
     - patch.needs_restarting
+    - ntp.soc_ntp
+    - ntp.adv_ntp
     - logrotate
+    - docker.soc_docker
+    - docker.adv_docker
+    - sensoroni.soc_sensoroni
+    - sensoroni.adv_sensoroni
+    - telegraf.soc_telegraf
+    - telegraf.adv_telegraf
+    - influxdb.token
+    - node_data.ips
 
-  '*_eval or *_helixsensor or *_heavynode or *_sensor or *_standalone or *_import':
+  '* and not *_eval and not *_import':
+    - logstash.nodes
+
+  '*_eval or *_heavynode or *_sensor or *_standalone or *_import':
     - match: compound
     - zeek
+    - bpf.soc_bpf
+    - bpf.adv_bpf
 
   '*_managersearch or *_heavynode':
     - match: compound
     - logstash
     - logstash.manager
     - logstash.search
-    - elasticsearch.search
+    - logstash.soc_logstash
+    - logstash.adv_logstash
+    - elasticsearch.index_templates
+    - elasticsearch.soc_elasticsearch
+    - elasticsearch.adv_elasticsearch
 
   '*_manager':
     - logstash
     - logstash.manager
-    - elasticsearch.manager
+    - logstash.soc_logstash
+    - logstash.adv_logstash
+    - elasticsearch.index_templates
 
   '*_manager or *_managersearch':
     - match: compound
-    - data.*
+    {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
+    - elasticsearch.auth
+    {% endif %}
+    {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/kibana/secrets.sls') %}
+    - kibana.secrets
+    {% endif %}
     - secrets
-    - global
+    - global.soc_global
+    - global.adv_global
+    - manager.soc_manager
+    - manager.adv_manager
+    - idstools.soc_idstools
+    - idstools.adv_idstools
+    - soc.soc_soc
+    - soc.adv_soc
+    - kratos.soc_kratos
+    - kratos.adv_kratos
+    - redis.soc_redis
+    - redis.adv_redis
+    - influxdb.soc_influxdb
+    - influxdb.adv_influxdb
+    - elasticsearch.soc_elasticsearch
+    - elasticsearch.adv_elasticsearch
+    - backup.soc_backup
+    - backup.adv_backup
+    - firewall.soc_firewall
+    - firewall.adv_firewall
     - minions.{{ grains.id }}
+    - minions.adv_{{ grains.id }}
 
   '*_sensor':
-    - zeeklogs
     - healthcheck.sensor
-    - global
+    - global.soc_global
+    - global.adv_global
     - minions.{{ grains.id }}
+    - minions.adv_{{ grains.id }}
 
   '*_eval':
-    - data.*
-    - zeeklogs
     - secrets
     - healthcheck.eval
-    - elasticsearch.eval
-    - global
+    - elasticsearch.index_templates
+    {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
+    - elasticsearch.auth
+    {% endif %}
+    {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/kibana/secrets.sls') %}
+    - kibana.secrets
+    {% endif %}
+    - global.soc_global
+    - global.adv_global
+    - kratos.soc_kratos
+    - elasticsearch.soc_elasticsearch
+    - elasticsearch.adv_elasticsearch
+    - manager.soc_manager
+    - manager.adv_manager
+    - idstools.soc_idstools
+    - idstools.adv_idstools
+    - soc.soc_soc
+    - kratos.soc_kratos
+    - kratos.adv_kratos
+    - redis.soc_redis
+    - redis.adv_redis
+    - influxdb.soc_influxdb
+    - influxdb.adv_influxdb
+    - backup.soc_backup
+    - backup.adv_backup
+    - firewall.soc_firewall
+    - firewall.adv_firewall
     - minions.{{ grains.id }}
+    - minions.adv_{{ grains.id }}
 
   '*_standalone':
     - logstash
     - logstash.manager
     - logstash.search
-    - elasticsearch.search
-    - data.*
-    - zeeklogs
+    - logstash.soc_logstash
+    - logstash.adv_logstash
+    - elasticsearch.index_templates
+    {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
+    - elasticsearch.auth
+    {% endif %}
+    {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/kibana/secrets.sls') %}
+    - kibana.secrets
+    {% endif %}
     - secrets
     - healthcheck.standalone
-    - global
-    - minions.{{ grains.id }}
-
-  '*_node':
-    - global
+    - global.soc_global
+    - global.adv_global
+    - idstools.soc_idstools
+    - idstools.adv_idstools
+    - kratos.soc_kratos
+    - kratos.adv_kratos
+    - redis.soc_redis
+    - redis.adv_redis
+    - influxdb.soc_influxdb
+    - influxdb.adv_influxdb
+    - elasticsearch.soc_elasticsearch
+    - elasticsearch.adv_elasticsearch
+    - manager.soc_manager
+    - manager.adv_manager
+    - soc.soc_soc
+    - backup.soc_backup
+    - backup.adv_backup
+    - firewall.soc_firewall
+    - firewall.adv_firewall
     - minions.{{ grains.id }}
+    - minions.adv_{{ grains.id }}
 
   '*_heavynode':
-    - zeeklogs
-    - global
+    - elasticsearch.auth
+    - global.soc_global
+    - global.adv_global
+    - redis.soc_redis
     - minions.{{ grains.id }}
+    - minions.adv_{{ grains.id }}
 
-  '*_helixsensor':
-    - fireeye
-    - zeeklogs
-    - logstash
-    - logstash.helix
-    - global
-    - minions.{{ grains.id }}
-
-  '*_fleet':
-    - data.*
-    - secrets
-    - global
+  '*_idh':
+    - global.soc_global
+    - global.adv_global
+    - idh.soc_idh
+    - idh.adv_idh
     - minions.{{ grains.id }}
+    - minions.adv_{{ grains.id }}
 
   '*_searchnode':
     - logstash
     - logstash.search
-    - elasticsearch.search
-    - global
+    - logstash.soc_logstash
+    - logstash.adv_logstash
+    - elasticsearch.index_templates
+    - elasticsearch.soc_elasticsearch
+    - elasticsearch.adv_elasticsearch
+    {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
+    - elasticsearch.auth
+    {% endif %}
+    - redis.soc_redis
+    - global.soc_global
+    - global.adv_global
     - minions.{{ grains.id }}
-    - data.nodestab
+    - minions.adv_{{ grains.id }}
+
+  '*_receiver':
+    - logstash
+    - logstash.receiver
+    - logstash.soc_logstash
+    - logstash.adv_logstash
+    {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
+    - elasticsearch.auth
+    {% endif %}
+    - redis.soc_redis
+    - redis.adv_redis
+    - global.soc_global
+    - global.adv_global
+    - minions.{{ grains.id }}
+    - minions.adv_{{ grains.id }}
 
   '*_import':
-    - zeeklogs
     - secrets
-    - elasticsearch.eval
-    - global
-    - minions.{{ grains.id }}
+    - elasticsearch.index_templates
+    {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
+    - elasticsearch.auth
+    {% endif %}
+    {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/kibana/secrets.sls') %}
+    - kibana.secrets
+    {% endif %}
+    - kratos.soc_kratos
+    - elasticsearch.soc_elasticsearch
+    - elasticsearch.adv_elasticsearch
+    - manager.soc_manager
+    - manager.adv_manager
+    - soc.soc_soc
+    - global.soc_global
+    - global.adv_global
+    - backup.soc_backup
+    - backup.adv_backup
+    - kratos.soc_kratos
+    - kratos.adv_kratos
+    - redis.soc_redis
+    - redis.adv_redis
+    - influxdb.soc_influxdb
+    - influxdb.adv_influxdb
+    - firewall.soc_firewall
+    - firewall.adv_firewall
+    - minions.{{ grains.id }}
+    - minions.adv_{{ grains.id }}
+
+  '*_fleet':
+    - global.soc_global
+    - global.adv_global
+    - backup.soc_backup
+    - backup.adv_backup
+    - logstash
+    - logstash.fleet
+    - logstash.soc_logstash
+    - logstash.adv_logstash
+    - minions.{{ grains.id }}
+    - minions.adv_{{ grains.id }}
+
+  '*_workstation':
+    - minions.{{ grains.id }}
+    - minions.adv_{{ grains.id }}

pillar/zeek/init.sls

@@ -1,56 +1 @@
 zeek:
-  zeekctl:
-    MailTo: root@localhost
-    MailConnectionSummary: 1
-    MinDiskSpace: 5
-    MailHostUpDown: 1
-    LogRotationInterval: 3600
-    LogExpireInterval: 0
-    StatsLogEnable: 1
-    StatsLogExpireInterval: 0
-    StatusCmdShowAll: 0
-    CrashExpireInterval: 0
-    SitePolicyScripts: local.zeek
-    LogDir: /nsm/zeek/logs
-    SpoolDir: /nsm/zeek/spool
-    CfgDir: /opt/zeek/etc
-    CompressLogs: 1
-  local:
-    '@load':
-      - misc/loaded-scripts
-      - tuning/defaults
-      - misc/capture-loss
-      - misc/stats
-      - frameworks/software/vulnerable
-      - frameworks/software/version-changes
-      - protocols/ftp/software
-      - protocols/smtp/software
-      - protocols/ssh/software
-      - protocols/http/software
-      - protocols/dns/detect-external-names
-      - protocols/ftp/detect
-      - protocols/conn/known-hosts
-      - protocols/conn/known-services
-      - protocols/ssl/known-certs
-      - protocols/ssl/validate-certs
-      - protocols/ssl/log-hostcerts-only
-      - protocols/ssh/geo-data
-      - protocols/ssh/detect-bruteforcing
-      - protocols/ssh/interesting-hostnames
-      - protocols/http/detect-sqli
-      - frameworks/files/hash-all-files
-      - frameworks/files/detect-MHR
-      - policy/frameworks/notice/extend-email/hostnames
-      - ja3
-      - hassh
-      - intel
-      - cve-2020-0601
-      - securityonion/bpfconf
-      - securityonion/communityid
-      - securityonion/file-extraction
-    '@load-sigs':
-      - frameworks/signatures/detect-windows-shells
-    redef:
-      - LogAscii::use_json = T;
-      - LogAscii::json_timestamps = JSON::TS_ISO8601;
-      - CaptureLoss::watch_interval = 5 mins;

salt/_modules/needs_restarting.py

@@ -10,7 +10,7 @@ def check():
     if path.exists('/var/run/reboot-required'):
       retval = 'True'
 
-  elif os == 'CentOS':
+  elif os == 'Rocky':
     cmd = 'needs-restarting -r > /dev/null 2>&1'
     
     try:

salt/_modules/so.py

@@ -5,6 +5,8 @@ import logging
 def status():
     return __salt__['cmd.run']('/usr/sbin/so-status')
 
+def version():
+    return __salt__['cp.get_file_str']('/etc/soversion')
 
 def mysql_conn(retry):
     log = logging.getLogger(__name__)
@@ -61,4 +63,4 @@ def mysql_conn(retry):
         for addr in ip_arr:
             log.debug(f'  - {addr}')
 
-    return mysql_up
+    return mysql_up

salt/airgap/init.sls

@@ -1,60 +0,0 @@
-{% set MANAGER = salt['grains.get']('master') %}
-airgapyum:
-  file.managed:
-    - name: /etc/yum/yum.conf
-    - source: salt://airgap/files/yum.conf
-
-airgap_repo:
-  pkgrepo.managed:
-    - humanname: Airgap Repo
-    - baseurl: https://{{ MANAGER }}/repo
-    - gpgcheck: 0
-    - sslverify: 0
-
-agbase:
-  file.absent:
-    - name: /etc/yum.repos.d/CentOS-Base.repo
-
-agcr:
-  file.absent:
-    - name: /etc/yum.repos.d/CentOS-CR.repo
-
-agdebug:
-  file.absent:
-    - name: /etc/yum.repos.d/CentOS-Debuginfo.repo
-
-agfasttrack:
-  file.absent:
-    - name: /etc/yum.repos.d/CentOS-fasttrack.repo
-
-agmedia:
-  file.absent:
-    - name: /etc/yum.repos.d/CentOS-Media.repo
-
-agsources:
-  file.absent:
-    - name: /etc/yum.repos.d/CentOS-Sources.repo
-
-agvault:
-  file.absent:
-    - name: /etc/yum.repos.d/CentOS-Vault.repo
-
-agkernel:
-  file.absent:
-    - name: /etc/yum.repos.d/CentOS-x86_64-kernel.repo
-
-agepel:
-  file.absent:
-    - name: /etc/yum.repos.d/epel.repo
-
-agtesting:
-  file.absent:
-    - name: /etc/yum.repos.d/epel-testing.repo
-
-agssrepo:
-  file.absent:
-    - name: /etc/yum.repos.d/saltstack.repo
-
-agwazrepo:
-  file.absent:
-    - name: /etc/yum.repos.d/wazuh.repo

salt/allowed_states.map.jinja

@@ -0,0 +1,280 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+
+{% set ZEEKVER = salt['pillar.get']('global:mdengine', '') %}
+{% set PLAYBOOK = salt['pillar.get']('manager:playbook', '0') %}
+{% set ELASTALERT = salt['pillar.get']('elastalert:enabled', True) %}
+{% set ELASTICSEARCH = salt['pillar.get']('elasticsearch:enabled', True) %}
+{% set KIBANA = salt['pillar.get']('kibana:enabled', True) %}
+{% set LOGSTASH = salt['pillar.get']('logstash:enabled', True) %}
+{% set CURATOR = salt['pillar.get']('curator:enabled', True) %}
+{% set REDIS = salt['pillar.get']('redis:enabled', True) %}
+{% set STRELKA = salt['pillar.get']('strelka:enabled', '0') %}
+{% set ISAIRGAP = salt['pillar.get']('global:airgap', False) %}
+{% import_yaml 'salt/minion.defaults.yaml' as saltversion %}
+{% set saltversion = saltversion.salt.minion.version %}
+
+{# this is the list we are returning from this map file, it gets built below #}
+{% set allowed_states= [] %}
+
+{% if grains.saltversion | string == saltversion | string %}
+
+  {% set allowed_states= salt['grains.filter_by']({
+      'so-eval': [
+          'salt.master',
+          'ca',
+          'ssl',
+          'registry',
+          'manager',
+          'nginx',
+          'telegraf',
+          'influxdb',
+          'soc',
+          'kratos',
+          'elasticfleet',
+          'firewall',
+          'idstools',
+          'suricata.manager',
+          'healthcheck',
+          'pcap',
+          'suricata',
+          'utility',
+          'schedule',
+          'soctopus',
+          'tcpreplay',
+          'docker_clean'
+          ],
+      'so-heavynode': [
+          'ssl',
+          'nginx',
+          'telegraf',
+          'firewall',
+          'pcap',
+          'suricata',
+          'healthcheck',
+          'schedule',
+          'tcpreplay',
+          'docker_clean'
+          ],
+      'so-helixsensor': [
+          'salt.master',
+          'ca',
+          'ssl',
+          'registry',
+          'telegraf',
+          'firewall',
+          'idstools',
+          'suricata.manager',
+          'zeek',
+          'redis',
+          'elasticsearch',
+          'logstash',
+          'schedule',
+          'tcpreplay',
+          'docker_clean'
+          ],
+     'so-idh': [
+          'ssl',
+          'telegraf',
+          'firewall',
+          'idh',
+          'schedule',
+          'docker_clean'
+          ],
+      'so-import': [
+          'salt.master',
+          'ca',
+          'ssl',
+          'registry',
+          'manager',
+          'nginx',
+          'soc',
+          'kratos',
+          'influxdb',
+          'telegraf',
+          'firewall',
+          'idstools',
+          'suricata.manager',
+          'pcap',
+          'utility',
+          'suricata',
+          'zeek',
+          'schedule',
+          'tcpreplay',
+          'docker_clean',
+          'elasticfleet'
+          ],
+      'so-manager': [
+          'salt.master',
+          'ca',
+          'ssl',
+          'registry',
+          'manager',
+          'nginx',
+          'telegraf',
+          'influxdb',
+          'soc',
+          'kratos',
+          'elasticfleet',
+          'firewall',
+          'idstools',
+          'suricata.manager',
+          'utility',
+          'schedule',
+          'soctopus',
+          'docker_clean'
+          ],
+      'so-managersearch': [
+          'salt.master',
+          'ca',
+          'ssl',
+          'registry',
+          'nginx',
+          'telegraf',
+          'influxdb',
+          'soc',
+          'kratos',
+          'elasticfleet',
+          'firewall',
+          'manager',
+          'idstools',
+          'suricata.manager',
+          'utility',
+          'schedule',
+          'soctopus',
+          'docker_clean'
+          ],
+      'so-searchnode': [
+          'ssl',
+          'nginx',
+          'telegraf',
+          'firewall',
+          'schedule',
+          'docker_clean'
+          ],
+      'so-standalone': [
+          'salt.master',
+          'ca',
+          'ssl',
+          'registry',
+          'manager',
+          'nginx',
+          'telegraf',
+          'influxdb',
+          'soc',
+          'kratos',
+          'elasticfleet',
+          'firewall',
+          'idstools',
+          'suricata.manager',
+          'pcap',
+          'suricata',
+          'healthcheck',
+          'utility',
+          'schedule',
+          'soctopus',
+          'tcpreplay',
+          'docker_clean'
+          ],
+      'so-sensor': [
+          'ssl',
+          'telegraf',
+          'firewall',
+          'nginx',
+          'pcap',
+          'suricata',
+          'healthcheck',
+          'schedule',
+          'tcpreplay',
+          'docker_clean'
+          ],
+      'so-fleet': [
+          'ssl',
+          'telegraf',
+          'firewall',
+          'logstash',
+          'healthcheck',
+          'schedule',
+          'elasticfleet',
+          'docker_clean'
+          ],
+      'so-receiver': [
+          'ssl',
+          'telegraf',
+          'firewall',
+          'schedule',
+          'docker_clean'
+          ],
+      'so-workstation': [
+          ],
+  }, grain='role') %}
+
+  {% if (PLAYBOOK != 0) and grains.role in ['so-eval', 'so-manager', 'so-managersearch', 'so-standalone'] %}
+    {% do allowed_states.append('mysql') %}
+  {% endif %}
+
+  {%- if ZEEKVER != 'SURICATA' and grains.role in ['so-sensor', 'so-eval', 'so-standalone', 'so-heavynode'] %}
+    {% do allowed_states.append('zeek') %}
+  {%- endif %}
+
+  {% if STRELKA and grains.role in ['so-sensor', 'so-eval', 'so-standalone', 'so-heavynode'] %}
+    {% do allowed_states.append('strelka') %}
+  {% endif %}
+
+  {% if ELASTICSEARCH and grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-searchnode', 'so-managersearch', 'so-heavynode', 'so-import'] %}
+    {% do allowed_states.append('elasticsearch') %}
+  {% endif %}
+
+  {% if ELASTICSEARCH and grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-managersearch', 'so-import'] %}
+    {% do allowed_states.append('elasticsearch.auth') %}
+  {% endif %}
+
+  {% if KIBANA and grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-managersearch', 'so-import'] %}
+    {% do allowed_states.append('kibana') %}
+    {% do allowed_states.append('kibana.secrets') %}
+  {% endif %}
+
+  {% if grains.role in ['so-eval', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-manager'] %}
+    {% do allowed_states.append('curator') %}
+  {% endif %}
+
+  {% if ELASTALERT and grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-managersearch'] %}
+    {% do allowed_states.append('elastalert') %}
+  {% endif %}
+
+  {% if (PLAYBOOK !=0) and grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-managersearch'] %}
+    {% do allowed_states.append('playbook') %}
+  {% endif %}
+
+  {% if (PLAYBOOK !=0) and grains.role in ['so-eval'] %}
+    {% do allowed_states.append('redis') %}
+  {% endif %}
+
+  {% if LOGSTASH and grains.role in ['so-helixsensor', 'so-manager', 'so-standalone', 'so-searchnode', 'so-managersearch', 'so-heavynode', 'so-receiver'] %}
+    {% do allowed_states.append('logstash') %}
+  {% endif %}
+
+  {% if REDIS and grains.role in ['so-manager', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-receiver'] %}
+    {% do allowed_states.append('redis') %}
+  {% endif %}
+  
+  {# all nodes on the right salt version can run the following states #}
+  {% do allowed_states.append('common') %}
+  {% do allowed_states.append('patch.os.schedule') %}
+  {% do allowed_states.append('motd') %}
+  {% do allowed_states.append('salt.minion-check') %}
+  {% do allowed_states.append('sensoroni') %}
+  {% do allowed_states.append('salt.lasthighstate') %}
+
+{% endif %}
+
+
+{% if ISAIRGAP %}
+  {% do allowed_states.append('airgap') %}
+{% endif %}
+
+{# all nodes can always run salt.minion state #}
+{% do allowed_states.append('salt.minion') %}

salt/backup/config_backup.sls

@@ -0,0 +1,34 @@
+{% from 'backup/map.jinja' import BACKUP_MERGED %}
+
+# Lock permissions on the backup directory
+backupdir:
+  file.directory:
+    - name: /nsm/backup
+    - user: 0
+    - group: 0
+    - makedirs: True
+    - mode: 700
+
+config_backup_script:
+  file.managed:
+    - name: /usr/sbin/so-config-backup
+    - user: root
+    - group: root
+    - mode: 755
+    - template: jinja
+    - source: salt://backup/tools/sbin/so-config-backup.jinja
+    - defaults:
+        BACKUPLOCATIONS: {{ BACKUP_MERGED.locations }}
+        DESTINATION: {{ BACKUP_MERGED.destination }}
+  
+# Add config backup
+so_config_backup:
+  cron.present:
+    - name: /usr/sbin/so-config-backup > /dev/null 2>&1
+    - identifier: so_config_backup
+    - user: root
+    - minute: '1'
+    - hour: '0'
+    - daymonth: '*'
+    - month: '*'
+    - dayweek: '*'

salt/backup/defaults.yaml

@@ -0,0 +1,7 @@
+backup:
+  locations:
+    - /opt/so/saltstack/local
+    - /etc/pki
+    - /etc/salt
+    - /nsm/kratos
+  destination: "/nsm/backup"

salt/backup/map.jinja

@@ -0,0 +1,2 @@
+{% import_yaml 'backup/defaults.yaml' as BACKUP_DEFAULTS %}
+{% set BACKUP_MERGED = salt['pillar.get']('backup', BACKUP_DEFAULTS.backup, merge=true, merge_nested_lists=true) %}

salt/backup/soc_backup.yaml

@@ -0,0 +1,10 @@
+backup:
+  locations:
+    description: List of locations to back up to the destination.
+    helpLink: backup.html
+    global: True
+  destination:
+    description: Directory to store the configuration backups in.
+    helpLink: backup.html
+    global: True
+    

salt/backup/tools/sbin/so-config-backup.jinja

@@ -0,0 +1,37 @@
+#!/bin/bash
+#
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+. /usr/sbin/so-common
+
+TODAY=$(date '+%Y_%m_%d')
+BACKUPDIR={{ DESTINATION }}
+BACKUPFILE="$BACKUPDIR/so-config-backup-$TODAY.tar"
+MAXBACKUPS=7
+
+# Create backup dir if it does not exist
+mkdir -p /nsm/backup
+
+# If we haven't already written a backup file for today, let's do so
+if [ ! -f $BACKUPFILE ]; then
+
+  # Create empty backup file
+  tar -cf $BACKUPFILE -T /dev/null
+
+  # Loop through all paths defined in global.sls, and append them to backup file
+  {%- for LOCATION in BACKUPLOCATIONS %}
+  tar -rf $BACKUPFILE {{ LOCATION }}
+  {%- endfor %}
+
+fi
+
+# Find oldest backup files and remove them
+NUMBACKUPS=$(find /nsm/backup/ -type f -name "so-config-backup*" | wc -l)
+while [ "$NUMBACKUPS" -gt "$MAXBACKUPS" ]; do
+  OLDESTBACKUP=$(find /nsm/backup/ -type f -name "so-config-backup*" -type f -printf '%T+ %p\n' | sort | head -n 1 | awk -F" " '{print $2}')
+  rm -f $OLDESTBACKUP
+  NUMBACKUPS=$(find /nsm/backup/ -type f -name "so-config-backup*" | wc -l)
+done

salt/bpf/defaults.yaml

@@ -0,0 +1,4 @@
+bpf:
+  pcap: []
+  suricata: []
+  zeek: []

salt/bpf/pcap.map.jinja

@@ -0,0 +1,4 @@
+{% import_yaml 'bpf/defaults.yaml' as BPFDEFAULTS %}
+{% set BPFMERGED = salt['pillar.get']('bpf', BPFDEFAULTS.bpf, merge=True) %}
+
+{% set PCAPBPF = BPFMERGED.pcap %}

salt/bpf/soc_bpf.yaml

@@ -0,0 +1,16 @@
+bpf:
+  pcap:
+    description: List of BPF filters to apply to PCAP.
+    multiline: True
+    forcedType: "[]string"
+    helpLink: bpf.html
+  suricata:
+    description: List of BPF filters to apply to Suricata.
+    multiline: True
+    forcedType: "[]string"
+    helpLink: bpf.html
+  zeek:
+    description: List of BPF filters to apply to Zeek.
+    multiline: True
+    forcedType: "[]string"
+    helpLink: bpf.html

salt/bpf/suricata.map.jinja

@@ -0,0 +1,4 @@
+{% import_yaml 'bpf/defaults.yaml' as BPFDEFAULTS %}
+{% set BPFMERGED = salt['pillar.get']('bpf', BPFDEFAULTS.bpf, merge=True) %}
+
+{% set SURICATABPF = BPFMERGED.suricata %}

salt/bpf/zeek.map.jinja

@@ -0,0 +1,4 @@
+{% import_yaml 'bpf/defaults.yaml' as BPFDEFAULTS %}
+{% set BPFMERGED = salt['pillar.get']('bpf', BPFDEFAULTS.bpf, merge=True) %}
+
+{% set ZEEKBPF = BPFMERGED.zeek %}

salt/ca/dirs.sls

@@ -0,0 +1,4 @@
+pki_issued_certs:
+  file.directory:
+    - name: /etc/pki/issued_certs
+    - makedirs: True

salt/ca/files/signing_policies.conf

@@ -1,3 +1,6 @@
+mine_functions:
+  x509.get_pem_entries: [/etc/pki/ca.crt]
+
 x509_signing_policies:
   filebeat:
     - minions: '*'
@@ -54,7 +57,7 @@ x509_signing_policies:
     - extendedKeyUsage: serverAuth
     - days_valid: 820
     - copypath: /etc/pki/issued_certs/
-  fleet:
+  elasticfleet:
     - minions: '*'
     - signing_private_key: /etc/pki/ca.key
     - signing_cert: /etc/pki/ca.crt
@@ -62,9 +65,8 @@ x509_signing_policies:
     - ST: Utah
     - L: Salt Lake City
     - basicConstraints: "critical CA:false"
-    - keyUsage: "critical keyEncipherment"
+    - keyUsage: "digitalSignature, nonRepudiation"
     - subjectKeyIdentifier: hash
     - authorityKeyIdentifier: keyid,issuer:always
-    - extendedKeyUsage: serverAuth
     - days_valid: 820
     - copypath: /etc/pki/issued_certs/

salt/ca/init.sls

@@ -1,23 +1,24 @@
-{% set show_top = salt['state.show_top']() %}
-{% set top_states = show_top.values() | join(', ') %}
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
 
-{% if 'ca' in top_states %}
+{% from 'allowed_states.map.jinja' import allowed_states %}
+{% if sls in allowed_states %}
+{% from 'vars/globals.map.jinja' import GLOBALS %}
+
+
+include:
+  - ca.dirs
 
-{% set manager = salt['grains.get']('master') %}
 /etc/salt/minion.d/signing_policies.conf:
   file.managed:
     - source: salt://ca/files/signing_policies.conf
 
-/etc/pki:
-  file.directory: []
-
-/etc/pki/issued_certs:
-  file.directory: []
-
 pki_private_key:
   x509.private_key_managed:
     - name: /etc/pki/ca.key
-    - bits: 4096
+    - keysize: 4096
     - passphrase:
     - cipher: aes_256_cbc
     - backup: True
@@ -26,10 +27,11 @@ pki_private_key:
       - x509: /etc/pki/ca.crt
     {%- endif %}
 
-/etc/pki/ca.crt:
+pki_public_ca_crt:
   x509.certificate_managed:
+    - name: /etc/pki/ca.crt
     - signing_private_key: /etc/pki/ca.key
-    - CN: {{ manager }}
+    - CN: {{ GLOBALS.manager }}
     - C: US
     - ST: Utah
     - L: Salt Lake City
@@ -37,19 +39,17 @@ pki_private_key:
     - keyUsage: "critical cRLSign, keyCertSign"
     - extendedkeyUsage: "serverAuth, clientAuth"
     - subjectKeyIdentifier: hash
-    - authorityKeyIdentifier: keyid,issuer:always
+    - authorityKeyIdentifier: keyid:always, issuer
     - days_valid: 3650
     - days_remaining: 0
     - backup: True
     - replace: False
     - require:
-      - file: /etc/pki
-
-x509_pem_entries:
-  module.run:
-    - mine.send:
-       - name: x509.get_pem_entries
-       - glob_path: /etc/pki/ca.crt
+      - sls: ca.dirs
+    - timeout: 30
+    - retry:
+        attempts: 5
+        interval: 30
 
 cakeyperms:
   file.managed:
@@ -60,8 +60,8 @@ cakeyperms:
 
 {% else %}
 
-ca_state_not_allowed:
+{{sls}}_state_not_allowed:
   test.fail_without_changes:
-    - name: ca_state_not_allowed
+    - name: {{sls}}_state_not_allowed
 
-{% endif %}
+{% endif %}

salt/ca/remove.sls

@@ -0,0 +1,7 @@
+pki_private_key:
+  file.absent:
+    - name: /etc/pki/ca.key
+
+pki_public_ca_crt:
+  file.absent:
+    - name: /etc/pki/ca.crt

salt/common/cron/common-rotate

@@ -1,2 +1,2 @@
 #!/bin/bash
-logrotate -f /opt/so/conf/log-rotate.conf >/dev/null 2>&1
+/usr/sbin/logrotate -f /opt/so/conf/log-rotate.conf > /dev/null 2>&1

salt/common/files/99-reserved-ports.conf

@@ -0,0 +1 @@
+net.ipv4.ip_local_reserved_ports=55000,57314,47760-47860

salt/common/files/daemon.json

@@ -1,12 +1,14 @@
 {%- set DOCKERRANGE = salt['pillar.get']('docker:range', '172.17.0.0/24') %}
 {%- set DOCKERBIND = salt['pillar.get']('docker:bip', '172.17.0.1/24') %}
 {
-    "registry-mirrors": [ "https://:5000" ],
-    "bip": "{{ DOCKERBIND }}",
-    "default-address-pools": [
-      {
-        "base" : "{{ DOCKERRANGE }}",
-        "size" : 24
-      }
-    ]
+  "registry-mirrors": [
+    "https://:5000"
+  ],
+  "bip": "{{ DOCKERBIND }}",
+  "default-address-pools": [
+    {
+      "base": "{{ DOCKERRANGE }}",
+      "size": 24
+    }
+  ]
 }

salt/common/files/log-rotate.conf

@@ -1,4 +1,6 @@
 {%- set logrotate_conf = salt['pillar.get']('logrotate:conf') %}
+{%- set group_conf = salt['pillar.get']('logrotate:group_conf') %}
+
 
 /opt/so/log/aptcacher-ng/*.log
 /opt/so/log/idstools/*.log
@@ -13,12 +15,23 @@
 /opt/so/log/fleet/*.log
 /opt/so/log/suricata/*.log
 /opt/so/log/mysql/*.log
-/opt/so/log/playbook/*.log
-/opt/so/log/logstash/*.log
-/opt/so/log/filebeat/*.log
 /opt/so/log/telegraf/*.log
 /opt/so/log/redis/*.log
+/opt/so/log/sensoroni/*.log
+/opt/so/log/stenographer/*.log
 /opt/so/log/salt/so-salt-minion-check
+/opt/so/log/salt/minion
+/opt/so/log/salt/master
+/opt/so/log/logscan/*.log
+/nsm/idh/*.log
 {
     {{ logrotate_conf | indent(width=4) }}
 }
+
+# Playbook's log directory needs additional configuration
+# because Playbook requires a more permissive directory
+/opt/so/log/playbook/*.log
+{
+    {{ logrotate_conf | indent(width=4) }}
+    {{ group_conf | indent(width=4) }}
+}

salt/common/files/sensor-rotate.conf

@@ -6,5 +6,17 @@
     nocompress
     create
     sharedscripts
-    endscript
 }
+
+/nsm/strelka/log/strelka.log
+{
+    daily
+    rotate 14
+    missingok
+    copytruncate
+    compress
+    create
+    extension .log
+    dateext
+    dateyesterday
+}

salt/common/files/soversion

@@ -0,0 +1,2 @@
+{%- set VERSION = salt['pillar.get']('global:soversion') -%}
+{{ VERSION }}

salt/common/files/vimrc

@@ -0,0 +1,5 @@
+" Activates filetype detection
+filetype plugin indent on
+
+" Sets .sls files to use YAML syntax highlighting
+autocmd BufNewFile,BufRead *.sls set syntax=yaml

salt/common/init.sls

@@ -1,9 +1,14 @@
-{% set show_top = salt['state.show_top']() %}
-{% set top_states = show_top.values() | join(', ') %}
+{% from 'allowed_states.map.jinja' import allowed_states %}
+{% if sls in allowed_states %}
 
-{% if 'common' in top_states %}
+{% from 'vars/globals.map.jinja' import GLOBALS %}
 
-{% set role = grains.id.split('_') | last %}
+include:
+  - common.soup_scripts
+  - common.packages
+{% if GLOBALS.role in GLOBALS.manager_roles %}
+  - manager.elasticsearch # needed for elastic_curl_config state
+{% endif %}
 
 # Remove variables.txt from /tmp - This is temp
 rmvariablesfile:
@@ -28,15 +33,15 @@ socore:
 soconfperms:
   file.directory:
     - name: /opt/so/conf
-    - uid: 939
-    - gid: 939
+    - user: 939
+    - group: 939
     - dir_mode: 770
 
 sostatusconf:
   file.directory:
     - name: /opt/so/conf/so-status
-    - uid: 939
-    - gid: 939
+    - user: 939
+    - group: 939
     - dir_mode: 770
 
 so-status.conf:
@@ -47,10 +52,15 @@ so-status.conf:
 sosaltstackperms:
   file.directory:
     - name: /opt/so/saltstack
-    - uid: 939
-    - gid: 939
+    - user: 939
+    - group: 939
     - dir_mode: 770
 
+so_log_perms:
+  file.directory:
+    - name: /opt/so/log
+    - dir_mode: 755
+
 # Create a state directory
 statedir:
   file.directory:
@@ -66,91 +76,12 @@ salttmp:
     - group: 939
     - makedirs: True
 
-# Install epel
-{% if grains['os'] == 'CentOS' %}
-repair_yumdb:
-  cmd.run:
-    - name: 'mv -f /var/lib/rpm/__db* /tmp && yum clean all'
-    - onlyif:
-      - 'yum check-update 2>&1 | grep "Error: rpmdb open failed"'
-
-epel:
-  pkg.installed:
-    - skip_suggestions: True
-    - pkgs:
-      - epel-release
-{% endif %}
-
-# Install common packages
-{% if grains['os'] != 'CentOS' %}     
-commonpkgs:
-  pkg.installed:
-    - skip_suggestions: True
-    - pkgs:
-      - apache2-utils
-      - wget
-      - ntpdate
-      - jq
-      - python3-docker
-      - docker-ce
-      - curl
-      - ca-certificates
-      - software-properties-common
-      - apt-transport-https
-      - openssl
-      - netcat
-      - python3-mysqldb
-      - sqlite3
-      - argon2
-      - libssl-dev
-      - python3-dateutil
-      - python3-m2crypto
-      - python3-mysqldb
-      - git
-heldpackages:
-  pkg.installed:
-    - pkgs:
-      - containerd.io: 1.2.13-2
-      - docker-ce: 5:19.03.14~3-0~ubuntu-bionic
-    - hold: True
-    - update_holds: True
-
-{% else %}
-commonpkgs:
-  pkg.installed:
-    - skip_suggestions: True
-    - pkgs:
-      - wget
-      - ntpdate
-      - bind-utils
-      - jq
-      - tcpdump
-      - httpd-tools
-      - net-tools
-      - curl
-      - sqlite
-      - argon2
-      - mariadb-devel
-      - nmap-ncat
-      - python3
-      - python36-docker
-      - python36-dateutil
-      - python36-m2crypto
-      - python36-mysql
-      - yum-utils
-      - device-mapper-persistent-data
-      - lvm2
-      - openssl
-      - git
-
-heldpackages:
-  pkg.installed:
-    - pkgs:
-      - containerd.io: 1.2.13-3.2.el7
-      - docker-ce: 3:19.03.14-3.el7
-    - hold: True
-    - update_holds: True
-{% endif %}
+# VIM config
+vimconfig:
+  file.managed:
+    - name: /root/.vimrc
+    - source: salt://common/files/vimrc
+    - replace: False
 
 # Always keep these packages up to date
 
@@ -166,6 +97,21 @@ alwaysupdated:
 Etc/UTC:
   timezone.system
 
+# Sync curl configuration for Elasticsearch authentication
+{% if GLOBALS.role in ['so-eval', 'so-heavynode', 'so-import', 'so-manager', 'so-managersearch', 'so-searchnode', 'so-standalone'] %}
+elastic_curl_config:
+  file.managed:
+    - name: /opt/so/conf/elasticsearch/curl.config
+    - source: salt://elasticsearch/curl.config
+    - mode: 600
+    - show_changes: False
+    - makedirs: True
+  {% if GLOBALS.role in GLOBALS.manager_roles %}
+    - require:
+      - file: elastic_curl_config_distributed
+  {% endif %}
+{% endif %}
+
 # Sync some Utilities
 utilsyncscripts:
   file.recurse:
@@ -175,11 +121,25 @@ utilsyncscripts:
     - file_mode: 755
     - template: jinja
     - source: salt://common/tools/sbin
+    - exclude_pat:
+        - so-common
+        - so-firewall
+        - so-image-common
+        - soup
+        - so-status
 
-{% if role in ['eval', 'standalone', 'sensor', 'heavynode'] %}
+so-status_script:
+  file.managed:
+    - name: /usr/sbin/so-status
+    - source: salt://common/tools/sbin/so-status
+    - mode: 755
+
+{% if GLOBALS.role in GLOBALS.sensor_roles %}
 # Add sensor cleanup
-/usr/sbin/so-sensor-clean:
+so-sensor-clean:
   cron.present:
+    - name: /usr/sbin/so-sensor-clean
+    - identifier: so-sensor-clean
     - user: root
     - minute: '*'
     - hour: '*'
@@ -199,8 +159,10 @@ sensorrotateconf:
     - source: salt://common/files/sensor-rotate.conf
     - mode: 644
 
-/usr/local/bin/sensor-rotate:
+sensor-rotate:
   cron.present:
+    - name: /usr/local/bin/sensor-rotate
+    - identifier: sensor-rotate
     - user: root
     - minute: '1'
     - hour: '0'
@@ -223,8 +185,10 @@ commonlogrotateconf:
     - template: jinja
     - mode: 644
 
-/usr/local/bin/common-rotate:
+common-rotate:
   cron.present:
+    - name: /usr/local/bin/common-rotate
+    - identifier: common-rotate
     - user: root
     - minute: '1'
     - hour: '0'
@@ -232,36 +196,76 @@ commonlogrotateconf:
     - month: '*'
     - dayweek: '*'
 
-{% if role in ['eval', 'manager', 'managersearch', 'standalone'] %}
-# Add config backup
-/usr/sbin/so-config-backup > /dev/null 2>&1:
+# Create the status directory
+sostatusdir:
+  file.directory:
+    - name: /opt/so/log/sostatus
+    - user: 0
+    - group: 0
+    - makedirs: True
+
+sostatus_log:
+  file.managed:
+    - name: /opt/so/log/sostatus/status.log
+    - mode: 644
+
+# Install sostatus check cron. This is used to populate Grid.
+so-status_check_cron:
   cron.present:
+    - name: '/usr/sbin/so-status -j > /opt/so/log/sostatus/status.log 2>&1'
+    - identifier: so-status_check_cron
     - user: root
-    - minute: '1'
-    - hour: '0'
+    - minute: '*/1'
+    - hour: '*'
     - daymonth: '*'
     - month: '*'
     - dayweek: '*'
+
+remove_post_setup_cron:
+  cron.absent:
+    - name: 'PATH=$PATH:/usr/sbin salt-call state.highstate'
+    - identifier: post_setup_cron
+
+{% if GLOBALS.role not in ['eval', 'manager', 'managersearch', 'standalone'] %}
+
+soversionfile:
+  file.managed:
+    - name: /etc/soversion
+    - source: salt://common/files/soversion
+    - mode: 644
+    - template: jinja
+    
 {% endif %}
 
-# Manager daemon.json
-docker_daemon:
-  file.managed:
-    - source: salt://common/files/daemon.json
-    - name: /etc/docker/daemon.json
-    - template: jinja 
+{% if GLOBALS.so_model and GLOBALS.so_model not in ['SO2AMI01', 'SO2AZI01', 'SO2GCI01'] %}
+  {% if GLOBALS.os == 'Rocky' %}     
+# Install Raid tools
+raidpkgs:
+  pkg.installed:
+    - skip_suggestions: True
+    - pkgs:
+      - securityonion-raidtools
+      - securityonion-megactl
+  {% endif %}
 
-# Make sure Docker is always running
-docker:
-  service.running:
-    - enable: True
-    - watch:
-      - file: docker_daemon
+# Install raid check cron
+so-raid-status:
+  cron.present:
+    - name: '/usr/sbin/so-raid-status > /dev/null 2>&1'
+    - identifier: so-raid-status
+    - user: root
+    - minute: '*/15'
+    - hour: '*'
+    - daymonth: '*'
+    - month: '*'
+    - dayweek: '*'
+
+{% endif %}
 
 {% else %}
 
-common_state_not_allowed:
+{{sls}}_state_not_allowed:
   test.fail_without_changes:
-    - name: common_state_not_allowed
+    - name: {{sls}}_state_not_allowed
 
 {% endif %}

salt/common/packages.sls

@@ -0,0 +1,67 @@
+{% from 'vars/globals.map.jinja' import GLOBALS %}
+
+{% if GLOBALS.os == 'Ubuntu' %}
+commonpkgs:
+  pkg.installed:
+    - skip_suggestions: True
+    - pkgs:
+      - apache2-utils
+      - wget
+      - ntpdate
+      - jq
+      - curl
+      - ca-certificates
+      - software-properties-common
+      - apt-transport-https
+      - openssl
+      - netcat
+      - sqlite3
+      - libssl-dev
+      - python3-dateutil
+      - python3-packaging
+      - python3-watchdog
+      - python3-lxml
+      - git
+      - vim
+
+# since Ubuntu requires and internet connection we can use pip to install modules
+python3-pip:
+  pkg.installed
+
+python-rich:
+  pip.installed:
+    - name: rich
+    - target: /usr/local/lib/python3.8/dist-packages/
+    - require:
+      - pkg: python3-pip
+  
+
+{% elif GLOBALS.os == 'Rocky' %}     
+commonpkgs:
+  pkg.installed:
+    - skip_suggestions: True
+    - pkgs:
+      - wget
+      - jq
+      - tcpdump
+      - httpd-tools
+      - net-tools
+      - curl
+      - sqlite
+      - mariadb-devel
+      - python3-dnf-plugin-versionlock
+      - nmap-ncat
+      - yum-utils
+      - device-mapper-persistent-data
+      - lvm2
+      - openssl
+      - git
+      - python3-docker
+      - python3-m2crypto
+      - rsync
+      - python3-rich
+      - python3-pyyaml
+      - python3-watchdog
+      - python3-packaging
+      - unzip
+{% endif %}

salt/common/soup_scripts.sls

@@ -0,0 +1,13 @@
+# Sync some Utilities
+soup_scripts:
+  file.recurse:
+    - name: /usr/sbin
+    - user: root
+    - group: root
+    - file_mode: 755
+    - source: salt://common/tools/sbin
+    - include_pat:
+        - so-common
+        - so-firewall
+        - so-image-common
+        - soup

salt/common/tools/sbin/so-allow

@@ -1,166 +1,11 @@
-#!/bin/bash
+#!/usr/bin/bash
 
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
 
-. /usr/sbin/so-common
+echo "Please use the Configuration section in SOC to allow hosts"
+echo ""
+echo "If you need command line options on adding hosts please run so-firewall"
 
-local_salt_dir=/opt/so/saltstack/local
-
-SKIP=0
-
-function usage {
-
-cat << EOF
-
-Usage: $0 [-abefhoprsw] [ -i IP ]
-
-This program allows you to add a firewall rule to allow connections from a new IP address or CIDR range.
-
-If you run this program with no arguments, it will present a menu for you to choose your options.
-
-If you want to automate and skip the menu, you can pass the desired options as command line arguments.
-
-EXAMPLES
-
-To add 10.1.2.3 to the analyst role:
-so-allow -a -i 10.1.2.3
-
-To add 10.1.2.0/24 to the osquery role:
-so-allow -o -i 10.1.2.0/24
-
-EOF
-
-}
-
-while getopts "ahfesprbowi:" OPTION
-do
-    case $OPTION in
-        h)
-            usage
-            exit 0
-            ;;
-        a)
-            FULLROLE="analyst"
-            SKIP=1
-            ;;
-        b)
-            FULLROLE="beats_endpoint"
-            SKIP=1
-            ;;
-	e)
-            FULLROLE="elasticsearch_rest"
-            SKIP=1
-            ;;
-        f)
-	    FULLROLE="strelka_frontend"
-            SKIP=1
-            ;;
-        i)  IP=$OPTARG
-            ;;
-        o)
-            FULLROLE="osquery_endpoint"
-            SKIP=1
-            ;;
-        w)
-            FULLROLE="wazuh_agent"
-            SKIP=1
-            ;;
-        s)
-            FULLROLE="syslog"
-            SKIP=1
-            ;;
-        p)
-            FULLROLE="wazuh_api"
-            SKIP=1
-            ;;
-        r)
-            FULLROLE="wazuh_authd"
-            SKIP=1
-            ;;
-        *)
-            usage
-            exit 0
-            ;;
-    esac
-done
-
-if [ "$SKIP" -eq 0 ]; then
-
-    echo "This program allows you to add a firewall rule to allow connections from a new IP address."
-    echo ""
-    echo "Choose the role for the IP or Range you would like to add"
-    echo ""
-    echo "[a] - Analyst - ports 80/tcp and 443/tcp"
-    echo "[b] - Logstash Beat - port 5044/tcp"
-    echo "[e] - Elasticsearch REST API - port 9200/tcp"
-    echo "[f] - Strelka frontend - port 57314/tcp"
-    echo "[o] - Osquery endpoint - port 8090/tcp"
-    echo "[s] - Syslog device - 514/tcp/udp"
-    echo "[w] - Wazuh agent - port 1514/tcp/udp"
-    echo "[p] - Wazuh API - port 55000/tcp"
-    echo "[r] - Wazuh registration service - 1515/tcp"
-    echo ""
-    echo "Please enter your selection:"
-    read -r ROLE
-    echo "Enter a single ip address or range to allow (example: 10.10.10.10 or 10.10.0.0/16):"
-    read -r IP
-
-    if [ "$ROLE" == "a" ]; then
-        FULLROLE=analyst
-    elif [ "$ROLE" == "b" ]; then
-        FULLROLE=beats_endpoint
-    elif [ "$ROLE" == "e" ]; then
-        FULLROLE=elasticsearch_rest
-    elif [ "$ROLE" == "f" ]; then
-        FULLROLE=strelka_frontend
-    elif [ "$ROLE" == "o" ]; then
-        FULLROLE=osquery_endpoint
-    elif [ "$ROLE" == "w" ]; then
-        FULLROLE=wazuh_agent
-    elif [ "$ROLE" == "s" ]; then
-        FULLROLE=syslog
-    elif [ "$ROLE" == "p" ]; then
-        FULLROLE=wazuh_api
-    elif [ "$ROLE" == "r" ]; then
-        FULLROLE=wazuh_authd
-    else
-        echo "I don't recognize that role"
-        exit 1
-    fi
-
-fi
-
-echo "Adding $IP to the $FULLROLE role. This can take a few seconds"
-/usr/sbin/so-firewall includehost $FULLROLE $IP
-salt-call state.apply firewall queue=True
-
-# Check if Wazuh enabled
-if grep -q -R "wazuh: 1" $local_salt_dir/pillar/*; then
-    # If analyst, add to Wazuh AR whitelist
-    if [ "$FULLROLE" == "analyst" ]; then
-        WAZUH_MGR_CFG="/nsm/wazuh/etc/ossec.conf"
-        if ! grep -q "<white_list>$IP</white_list>" $WAZUH_MGR_CFG ; then
-            DATE=$(date)
-            sed -i 's/<\/ossec_config>//' $WAZUH_MGR_CFG
-            sed -i '/^$/N;/^\n$/D' $WAZUH_MGR_CFG
-            echo -e "<!--Address $IP added by /usr/sbin/so-allow on \"$DATE\"-->\n  <global>\n    <white_list>$IP</white_list>\n  </global>\n</ossec_config>" >> $WAZUH_MGR_CFG
-            echo "Added whitelist entry for $IP in $WAZUH_MGR_CFG."
-        echo
-            echo "Restarting OSSEC Server..."
-            /usr/sbin/so-wazuh-restart
-        fi
-    fi
-fi

salt/common/tools/sbin/so-allow-view

@@ -1,23 +1,15 @@
 #!/bin/bash
 
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+
 
 . /usr/sbin/so-common
 
 echo ""
 echo "Hosts/Networks that have access to login to the Security Onion Console:"
 
-so-firewall includedhosts analyst
+so-firewall includedhosts analyst

salt/common/tools/sbin/so-analyst-install

@@ -1,309 +1,91 @@
 #!/bin/bash
 
-# Copyright 2014-2020 Security Onion Solutions, LLC
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
 
-#    This program is free software: you can redistribute it and/or modify
-#    it under the terms of the GNU General Public License as published by
-#    the Free Software Foundation, either version 3 of the License, or
-#    (at your option) any later version.
-#
-#    This program is distributed in the hope that it will be useful,
-#    but WITHOUT ANY WARRANTY; without even the implied warranty of
-#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#    GNU General Public License for more details.
-#
-#    You should have received a copy of the GNU General Public License
-#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-if [ "$(id -u)" -ne 0 ]; then
-	echo "This script must be run using sudo!"
-	exit 1
-fi
+{# we only want the script to install the workstation if it is Rocky -#}
+{% if grains.os == 'Rocky' -%}
+{#   if this is a manager -#}
+{%   if grains.master == grains.id.split('_')|first -%}
 
-INSTALL_LOG=/root/so-analyst-install.log
-exec &> >(tee -a "$INSTALL_LOG")
+source /usr/sbin/so-common
+doc_workstation_url="$DOC_BASE_URL/analyst-vm.html"
+pillar_file="/opt/so/saltstack/local/pillar/minions/{{grains.id}}.sls"
 
-log() {
-	msg=$1
-	level=${2:-I}
-	now=$(TZ=GMT date +"%Y-%m-%dT%H:%M:%SZ")
-	echo -e "$now | $level | $msg" >> "$INSTALL_LOG" 2>&1
-}
+if [ -f "$pillar_file" ]; then
+  if ! grep -q "^workstation:$" "$pillar_file"; then
 
-error() {
-	log "$1" "E"
-}
-
-info() {
-	log "$1" "I"
-}
-
-title() {
-	echo -e "\n-----------------------------\n $1\n-----------------------------\n" >> "$INSTALL_LOG" 2>&1
-}
-
-logCmd() {
-	cmd=$1
-	info "Executing command: $cmd"
-	$cmd >> "$INSTALL_LOG" 2>&1
-}
-
-analyze_system() {
-	title "System Characteristics"
-	logCmd "uptime"
-	logCmd "uname -a"
-	logCmd "free -h"
-	logCmd "lscpu"
-	logCmd "df -h"
-	logCmd "ip a"
-}
-
-analyze_system
-
-OS=$(grep PRETTY_NAME /etc/os-release | grep 'CentOS Linux 7')
-if [ $? -ne 0 ]; then
-  echo "This is an unsupported OS. Please use CentOS 7 to install the analyst node."
-  exit 1
-fi
-
-if [[ "$manufacturer" == "Security Onion Solutions" && "$family" == "Automated" ]]; then
-  INSTALL=yes
-  CURLCONTINUE=no
-else
-  INSTALL=''
-  CURLCONTINUE=''
-fi
-
-FIRSTPASS=yes
-while [[ $INSTALL != "yes" ]] && [[ $INSTALL != "no" ]]; do
-  if [[ "$FIRSTPASS" == "yes" ]]; then
-    clear
-    echo "###########################################"
-    echo "##          ** W A R N I N G **          ##"
-    echo "##    _______________________________    ##"
-    echo "##                                       ##"
-    echo "##    Installing the Security Onion      ##"
-    echo "##   analyst node on this device will    ##"
-    echo "##      make permanenet changes to       ##"
-    echo "##              the system.              ##"
-    echo "##                                       ##"
-    echo "###########################################"
-    echo "Do you wish to continue? (Type the entire word 'yes' to proceed or 'no' to exit)"
-    FIRSTPASS=no
-  else
-    echo "Please type 'yes' to continue or 'no' to exit."
-  fi      
-  read INSTALL
-done
-
-if [[ $INSTALL == "no" ]]; then
-  echo "Exiting analyst node installation."
-  exit 0
-fi
-
-echo "Testing for internet connection with curl https://securityonionsolutions.com/"
-CANCURL=$(curl -sI https://securityonionsolutions.com/ | grep "200 OK")
-  if [ $? -ne 0 ]; then
     FIRSTPASS=yes
-    while [[ $CURLCONTINUE != "yes" ]] && [[ $CURLCONTINUE != "no" ]]; do
+    while [[ $INSTALL != "yes" ]] && [[ $INSTALL != "no" ]]; do
       if [[ "$FIRSTPASS" == "yes" ]]; then
-        echo "We could not access https://securityonionsolutions.com/."
-        echo "Since packages are downloaded from the internet, internet acceess is required."
-        echo "If you would like to ignore this warning and continue anyway, please type 'yes'."
-        echo "Otherwise, type 'no' to exit."
+        echo "###########################################"
+        echo "##          ** W A R N I N G **          ##"
+        echo "##    _______________________________    ##"
+        echo "##                                       ##"
+        echo "##    Installing the Security Onion      ##"
+        echo "##   analyst node on this device will    ##"
+        echo "##       make permanent changes to       ##"
+        echo "##              the system.              ##"
+        echo "##    A system reboot will be required   ##"
+        echo "##        to complete the install.       ##"
+        echo "##                                       ##"
+        echo "###########################################"
+        echo "Do you wish to continue? (Type the entire word 'yes' to proceed or 'no' to exit)"
         FIRSTPASS=no
       else
         echo "Please type 'yes' to continue or 'no' to exit."
-      fi  
-      read CURLCONTINUE
+      fi      
+      read INSTALL
     done
-    if [[ "$CURLCONTINUE" == "no" ]]; then
+
+    if [[ $INSTALL == "no" ]]; then
       echo "Exiting analyst node installation."
       exit 0
     fi
-  else
-    echo "We were able to curl https://securityonionsolutions.com/."
-    sleep 3
+
+    # Add workstation pillar to the minion's pillar file
+    printf '%s\n'\
+      "workstation:"\
+      "  gui:"\
+      "    enabled: true"\
+		  "" >> "$pillar_file"
+    echo "Applying the workstation state. This could take some time since there are many packages that need to be installed."
+    if salt-call state.apply workstation -linfo queue=True; then # make sure the state ran successfully
+      echo ""
+      echo "Analyst workstation has been installed!"
+      echo "Press ENTER to reboot or Ctrl-C to cancel."
+      read pause
+
+      reboot;
+    else
+      echo "There was an issue applying the workstation state. Please review the log above or at /opt/so/log/salt/minion."
+    fi
+  else # workstation is already added
+    echo "The workstation pillar already exists in $pillar_file."
+    echo "To enable/disable the gui, set 'workstation:gui:enabled' to true or false in $pillar_file."
+    echo "Additional documentation can be found at $doc_workstation_url."
   fi
-
-# Install a GUI text editor
-yum -y install gedit
-
-# Install misc utils
-yum -y install wget curl unzip epel-release yum-plugin-versionlock;
-
-# Install xWindows
-yum -y groupinstall "X Window System";
-yum -y install gnome-classic-session gnome-terminal nautilus-open-terminal control-center liberation-mono-fonts;
-unlink /etc/systemd/system/default.target;
-ln -sf /lib/systemd/system/graphical.target /etc/systemd/system/default.target;
-yum -y install file-roller
-
-# Install Mono - prereq for NetworkMiner
-yum -y install mono-core mono-basic mono-winforms expect
-
-# Install NetworkMiner
-yum -y install libcanberra-gtk2;
-wget https://www.netresec.com/?download=NetworkMiner -O /tmp/nm.zip;
-mkdir -p /opt/networkminer/
-unzip /tmp/nm.zip -d /opt/networkminer/;
-rm /tmp/nm.zip;
-mv /opt/networkminer/NetworkMiner_*/* /opt/networkminer/
-chmod +x /opt/networkminer/NetworkMiner.exe;
-chmod -R go+w /opt/networkminer/AssembledFiles/;
-chmod -R go+w /opt/networkminer/Captures/;
-# Create networkminer shim
-cat << EOF >> /bin/networkminer
-#!/bin/bash
-/bin/mono /opt/networkminer/NetworkMiner.exe --noupdatecheck "\$@"
-EOF
-chmod +x /bin/networkminer
-# Convert networkminer ico file to png format
-yum -y install ImageMagick
-convert /opt/networkminer/networkminericon.ico /opt/networkminer/networkminericon.png
-# Create menu entry
-cat << EOF >> /usr/share/applications/networkminer.desktop
-[Desktop Entry]
-Name=NetworkMiner
-Comment=NetworkMiner
-Encoding=UTF-8
-Exec=/bin/networkminer %f
-Icon=/opt/networkminer/networkminericon-4.png
-StartupNotify=true
-Terminal=false
-X-MultipleArgs=false
-Type=Application
-MimeType=application/x-pcap;
-Categories=Network;
-EOF
-
-# Set default monospace font to Liberation
-cat << EOF >> /etc/fonts/local.conf
- <match target="pattern">
-  <test name="family" qual="any">
-   <string>monospace</string>
-  </test>
-  <edit binding="strong" mode="prepend" name="family">
-   <string>Liberation Mono</string>
-  </edit>
- </match>
-EOF
-
-# Install Wireshark for Gnome
-yum -y install wireshark-gnome; 
-
-# Install dnsiff
-yum -y install dsniff;
-
-# Install hping3
-yum -y install hping3;
-
-# Install netsed
-yum -y install netsed;
-
-# Install ngrep
-yum -y install ngrep;
-
-# Install scapy
-yum -y install python36-scapy;
-
-# Install ssldump
-yum -y install ssldump;
-
-# Install tcpdump
-yum -y install tcpdump;
-
-# Install tcpflow
-yum -y install tcpflow;
-
-# Install tcpxtract
-yum -y install tcpxtract;
-
-# Install whois
-yum -y install whois;
-
-# Install foremost
-yum -y install https://forensics.cert.org/centos/cert/7/x86_64//foremost-1.5.7-13.1.el7.x86_64.rpm;
-
-# Install chromium
-yum -y install chromium;
-
-# Install tcpstat
-yum -y install https://github.com/Security-Onion-Solutions/securityonion-docker-rpm/releases/download/securityonion-tcpstat-1.5.0/securityonion-tcpstat-1.5.0.rpm;
-
-# Install tcptrace
-yum -y install https://github.com/Security-Onion-Solutions/securityonion-docker-rpm/releases/download/securityonion-tcptrace-6.6.7/securityonion-tcptrace-6.6.7.rpm;
-
-# Install sslsplit
-yum -y install libevent;
-yum -y install sslsplit;
-
-# Install Bit-Twist
-yum -y install https://github.com/Security-Onion-Solutions/securityonion-docker-rpm/releases/download/securityonion-bittwist-2.0.0/securityonion-bittwist-2.0.0.rpm;
-
-# Install chaosreader
-yum -y install perl-IO-Compress perl-Net-DNS;
-yum -y install https://github.com/Security-Onion-Solutions/securityonion-docker-rpm/releases/download/securityonion-chaosreader-0.95.10/securityonion-chaosreader-0.95.10.rpm;
-chmod +x /bin/chaosreader;
-
-if [ -f ../../files/analyst/README ]; then
-  cp ../../files/analyst/README /;
-  cp ../../files/analyst/so-wallpaper.jpg /usr/share/backgrounds/;
-  cp ../../files/analyst/so-lockscreen.jpg /usr/share/backgrounds/;
-  cp ../../files/analyst/so-login-logo-dark.svg /usr/share/pixmaps/;
-else
-  cp /opt/so/saltstack/default/salt/common/files/analyst/README /;
-  cp /opt/so/saltstack/default/salt/common/files/analyst/so-wallpaper.jpg /usr/share/backgrounds/;
-  cp /opt/so/saltstack/default/salt/common/files/analyst/so-lockscreen.jpg /usr/share/backgrounds/;
-  cp /opt/so/saltstack/default/salt/common/files/analyst/so-login-logo-dark.svg /usr/share/pixmaps/;
+else # if the pillar file doesn't exist
+  echo "Could not find $pillar_file and add the workstation pillar."
 fi
 
-# Set background wallpaper
-cat << EOF >> /etc/dconf/db/local.d/00-background
-# Specify the dconf path
-[org/gnome/desktop/background]
+{#-  if this is not a manager #}
+{%   else -%}
 
-# Specify the path to the desktop background image file
-picture-uri='file:///usr/share/backgrounds/so-wallpaper.jpg'
-# Specify one of the rendering options for the background image:
-# 'none', 'wallpaper', 'centered', 'scaled', 'stretched', 'zoom', 'spanned'
-picture-options='zoom'
-# Specify the left or top color when drawing gradients or the solid color
-primary-color='000000'
-# Specify the right or bottom color when drawing gradients
-secondary-color='FFFFFF'
-EOF
+echo "Since this is not a manager, the pillar values to enable analyst workstation must be set manually. Please view the documentation at $doc_workstation_url."
 
-# Set lock screen
-cat << EOF >> /etc/dconf/db/local.d/00-screensaver
-[org/gnome/desktop/session]
-idle-delay=uint32 180
+{#- endif if this is a manager #}
+{%   endif -%}
 
-[org/gnome/desktop/screensaver]
-lock-enabled=true
-lock-delay=uint32 120
-picture-options='zoom'
-picture-uri='file:///usr/share/backgrounds/so-lockscreen.jpg'
-EOF
+{#- if not Rocky #}
+{%- else %}
 
-cat << EOF >> /etc/dconf/db/local.d/locks/screensaver
-/org/gnome/desktop/session/idle-delay
-/org/gnome/desktop/screensaver/lock-enabled
-/org/gnome/desktop/screensaver/lock-delay
-EOF
+echo "The Analyst Workstation can only be installed on Rocky. Please view the documentation at $doc_workstation_url."
 
-# Do not show the user list at login screen
-cat << EOF >> /etc/dconf/db/local.d/00-login-screen
-[org/gnome/login-screen]
-logo='/usr/share/pixmaps/so-login-logo-dark.svg'
-disable-user-list=true
-EOF
+{#- endif grains.os == Rocky #}
+{% endif -%}
 
-dconf update;
-
-echo
-echo "Analyst workstation has been installed!"
-echo "Press ENTER to reboot or Ctrl-C to cancel."
-read pause
-
-reboot;
+exit 0

salt/common/tools/sbin/so-bpf-compile

@@ -29,7 +29,7 @@ fi
 
 interface="$1"
 shift
-sudo tcpdump -i $interface -ddd $@ | tail -n+2 |
+tcpdump -i $interface -ddd $@ | tail -n+2 |
 while read line; do
   cols=( $line )
   printf "%04x%02x%02x%08x" ${cols[0]} ${cols[1]} ${cols[2]} ${cols[3]}

salt/common/tools/sbin/so-checkin

@@ -1,20 +1,12 @@
 #!/bin/bash
 
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+
 
 . /usr/sbin/so-common
 
-salt-call state.highstate
+salt-call state.highstate -l info 

salt/common/tools/sbin/so-common

@@ -1,71 +1,93 @@
 #!/bin/bash
 #
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
 
-# Check for prerequisites
-if [ "$(id -u)" -ne 0 ]; then
-  echo "This script must be run using sudo!"
-  exit 1
+DEFAULT_SALT_DIR=/opt/so/saltstack/default
+DOC_BASE_URL="https://docs.securityonion.net/en/2.4"
+
+if [ -z $NOROOT ]; then
+	# Check for prerequisites
+	if [ "$(id -u)" -ne 0 ]; then
+		echo "This script must be run using sudo!"
+		exit 1
+	fi
+fi
+
+# Ensure /usr/sbin is in path
+if ! echo "$PATH" | grep -q "/usr/sbin"; then
+  export PATH="$PATH:/usr/sbin"
 fi
 
 # Define a banner to separate sections
 banner="========================================================================="
 
-header() {
-	echo
-    printf '%s\n' "$banner" "$*" "$banner"
-}
+add_interface_bond0() {
+	local BNIC=$1
+	if [[ -z $MTU ]]; then
+		local MTU
+		MTU=$(lookup_pillar "mtu" "sensor")
+	fi
+	local nic_error=0
 
-lookup_salt_value() {
-  key=$1
-  group=$2
-  kind=$3
+	# Check if specific offload features are able to be disabled
+	for string in "generic-segmentation-offload" "generic-receive-offload" "tcp-segmentation-offload"; do
+		if ethtool -k "$BNIC" | grep $string | grep -q "on [fixed]"; then
+			echo "The hardware or driver for interface ${BNIC} is not supported, packet capture may not work as expected."
+			((nic_error++))
+			break
+		fi
+	done
 
-  if [ -z "$kind" ]; then
-    kind=pillar
-  fi
+	case "$2" in
+		-v|--verbose)
+			local verbose=true
+		;;
+	esac
 
-  if [ -n "$group" ]; then
-    group=${group}:
-  fi
+	for i in rx tx sg tso ufo gso gro lro; do
+		if [[ $verbose == true ]]; then
+			ethtool -K "$BNIC" $i off
+		else
+			ethtool -K "$BNIC" $i off &>/dev/null
+		fi
+	done
 
-  salt-call --no-color  ${kind}.get ${group}${key} --out=newline_values_only
-}
+        if ! [[ $is_cloud ]]; then
+	  # Check if the bond slave connection has already been created
+	  nmcli -f name,uuid -p con | grep -q "bond0-slave-$BNIC"
+	  local found_int=$?
 
-lookup_pillar() {
-  key=$1
-  pillar=$2
-  if [ -z "$pillar" ]; then
-    pillar=global
-  fi
-  lookup_salt_value "$key" "$pillar" "pillar"
-}
+	  if [[ $found_int != 0 ]]; then
+		  # Create the slave interface and assign it to the bond
+		  nmcli con add type ethernet ifname "$BNIC" con-name "bond0-slave-$BNIC" master bond0 -- \
+			  ethernet.mtu "$MTU" \
+			  connection.autoconnect "yes"
+	  else
+		  local int_uuid
+		  int_uuid=$(nmcli -f name,uuid -p con | sed -n "s/bond0-slave-$BNIC //p" | tr -d ' ')
 
-lookup_pillar_secret() {
-  lookup_pillar "$1" "secrets"
-}
+		  nmcli con mod "$int_uuid" \
+			  ethernet.mtu "$MTU" \
+			  connection.autoconnect "yes"
+	  fi
+        fi
 
-lookup_grain() {
-  lookup_salt_value "$1" "" "grains"
-}
+	ip link set dev "$BNIC" arp off multicast off allmulticast off promisc on
 
-lookup_role() {
-  id=$(lookup_grain id)
-  pieces=($(echo $id | tr '_' ' '))
-  echo ${pieces[1]}
+        if ! [[ $is_cloud ]]; then
+	  # Bring the slave interface up
+	  if [[ $verbose == true ]]; then
+		  nmcli con up "bond0-slave-$BNIC"
+	  else
+		  nmcli con up "bond0-slave-$BNIC" &>/dev/null
+	  fi
+	fi
+	if [ "$nic_error" != 0 ]; then
+		return "$nic_error"
+	fi
 }
 
 check_container() {
@@ -74,69 +96,574 @@ check_container() {
 }
 
 check_password() {
-  local password=$1
-  echo "$password" | egrep -v "'|\"|\\$|\\\\" > /dev/null 2>&1
-  return $?
+	local password=$1
+	echo "$password" | egrep -v "'|\"|\\$|\\\\" > /dev/null 2>&1
+	return $?
 }
 
-set_os() {
-  if [ -f /etc/redhat-release ]; then
-    OS=centos
-  else
-    OS=ubuntu
-  fi
+check_password_and_exit() {
+	local password=$1
+	if ! check_password "$password"; then
+		echo "Password is invalid. Do not include single quotes, double quotes, dollar signs, and backslashes in the password."
+		exit 2
+	fi
+	return 0
 }
 
-set_minionid() {
-  MINIONID=$(lookup_grain id)
+check_elastic_license() {
+
+	[ -n "$TESTING" ] && return
+
+	# See if the user has already accepted the license
+	if [ ! -f /opt/so/state/yeselastic.txt ]; then
+	  elastic_license
+	else
+	  echo "Elastic License has already been accepted"
+	fi  
 }
 
-set_version() {
-  CURRENTVERSION=0.0.0
-  if [ -f /etc/soversion ]; then
-    CURRENTVERSION=$(cat /etc/soversion)
-  fi
-  if [ -z "$VERSION" ]; then
-    if [ -z "$NEWVERSION" ]; then
-      if [ "$CURRENTVERSION" == "0.0.0" ]; then
-        echo "ERROR: Unable to detect Security Onion version; terminating script."
-        exit 1
-      else
-        VERSION=$CURRENTVERSION
-      fi
-    else
-      VERSION="$NEWVERSION"
-    fi
-  fi
+check_salt_master_status() {
+	local timeout=$1
+	echo "Checking if we can talk to the salt master"
+	salt-call state.show_top concurrent=true
+
+	return
 }
 
-require_manager() {
-  # Check to see if this is a manager
-  MANAGERCHECK=$(cat /etc/salt/grains | grep role | awk '{print $2}')
-  if [ $MANAGERCHECK == 'so-eval' ] || [ $MANAGERCHECK == 'so-manager' ] || [ $MANAGERCHECK == 'so-managersearch' ] || [ $MANAGERCHECK == 'so-standalone' ] || [ $MANAGERCHECK == 'so-helix' ] || [ $MANAGERCHECK == 'so-import' ]; then
-    echo "This is a manager, We can proceed."
-  else
-    echo "Please run this command on the manager; the manager controls the grid."
-    exit 1
-  fi
+check_salt_minion_status() {
+	local timeout=$1
+	echo "Checking if the salt minion will respond to jobs" >> "$setup_log" 2>&1
+	salt "$MINION_ID" test.ping -t $timeout > /dev/null 2>&1
+	local status=$?
+	if [ $status -gt 0 ]; then
+		echo "  Minion did not respond" >> "$setup_log" 2>&1
+	else
+		echo "  Received job response from salt minion" >> "$setup_log" 2>&1
+	fi
+
+	return $status
 }
 
-is_single_node_grid() {
-  role=$(lookup_role)
-  if [ "$role" != "eval" ] && [ "$role" != "standalone" ] && [ "$role" != "import" ]; then
-    return 1
-  fi
-  return 0
+
+
+copy_new_files() {
+  # Copy new files over to the salt dir
+  cd $UPDATE_DIR
+  rsync -a salt $DEFAULT_SALT_DIR/
+  rsync -a pillar $DEFAULT_SALT_DIR/
+  chown -R socore:socore $DEFAULT_SALT_DIR/
+  chmod 755 $DEFAULT_SALT_DIR/pillar/firewall/addfirewall.sh
+  cd /tmp
+}
+
+disable_fastestmirror() {
+	sed -i 's/enabled=1/enabled=0/' /etc/yum/pluginconf.d/fastestmirror.conf
+}
+
+elastic_fleet_integration_create() {
+
+    JSON_STRING=$1
+
+    curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
+}
+
+elastic_fleet_policy_create() {
+
+    NAME=$1
+    DESC=$2
+    FLEETSERVER=$3
+
+    JSON_STRING=$( jq -n \
+                    --arg NAME "$NAME" \
+                    --arg DESC "$DESC" \
+                    --arg FLEETSERVER "$FLEETSERVER" \
+                    '{"name": $NAME,"id":$NAME,"description":$DESC,"namespace":"default","monitoring_enabled":["logs"],"inactivity_timeout":1209600,"has_fleet_server":$FLEETSERVER}'
+                    )
+    # Create Fleet Policy
+    curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/agent_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" 
+
+}
+
+elastic_fleet_policy_update() {
+
+    POLICYID=$1
+    JSON_STRING=$2
+
+    curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/agent_policies/$POLICYID" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
+}
+
+
+elastic_license() {
+
+read -r -d '' message <<- EOM
+\n
+Elastic Stack binaries and Security Onion components are only available under the Elastic License version 2 (ELv2):
+https://securityonion.net/license/
+
+Do you agree to the terms of ELv2?
+
+If so, type AGREE to accept ELv2 and continue. Otherwise, press Enter to exit this program without making any changes.
+EOM
+
+	AGREED=$(whiptail --title "$whiptail_title" --inputbox \
+	"$message" 20 75 3>&1 1>&2 2>&3)
+
+	if [ "${AGREED^^}" = 'AGREE' ]; then
+		mkdir -p /opt/so/state
+		touch /opt/so/state/yeselastic.txt 
+	else
+		echo "Starting in 2.3.40 you must accept the Elastic license if you want to run Security Onion."
+		exit 1
+	fi
+
 }
 
 fail() {
-  msg=$1
-  echo "ERROR: $msg"
-  echo "Exiting."
-  exit 1
+	msg=$1
+	echo "ERROR: $msg"
+	echo "Exiting."
+	exit 1
 }
 
 get_random_value() {
-  length=${1:-20}
-  head -c 5000 /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w $length | head -n 1
-}
+	length=${1:-20}
+	head -c 5000 /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w $length | head -n 1
+}
+
+gpg_rpm_import() {
+	if [[ "$OS" == "rocky" ]]; then
+	    if [[ "$WHATWOULDYOUSAYYAHDOHERE" == "setup" ]]; then
+	        local RPMKEYSLOC="../salt/repo/client/files/rocky/keys"
+	    else
+	        local RPMKEYSLOC="$UPDATE_DIR/salt/repo/client/files/rocky/keys"
+	    fi
+	
+	    RPMKEYS=('RPM-GPG-KEY-EPEL-9' 'SALT-PROJECT-GPG-PUBKEY-2023.pub' 'docker.pub' 'securityonion.pub')
+
+	    for RPMKEY in "${RPMKEYS[@]}"; do
+    	        rpm --import $RPMKEYSLOC/$RPMKEY
+	        echo "Imported $RPMKEY"
+	    done
+	fi
+}
+
+header() {
+	printf '%s\n' "" "$banner" " $*" "$banner" 
+}
+
+init_monitor() {
+	MONITORNIC=$1
+	
+	if [[ $MONITORNIC == "bond0" ]]; then 
+	  BIFACES=$(lookup_bond_interfaces)
+	else
+	  BIFACES=$MONITORNIC
+	fi
+
+	for DEVICE_IFACE in $BIFACES; do
+	  for i in rx tx sg tso ufo gso gro lro; do
+	    ethtool -K "$DEVICE_IFACE" "$i" off;
+  	  done
+	  ip link set dev "$DEVICE_IFACE" arp off multicast off allmulticast off promisc on
+	done
+}
+
+is_manager_node() {
+	grep "role: so-" /etc/salt/grains | grep -E "manager|eval|managersearch|standalone|import" &> /dev/null
+}
+
+is_sensor_node() {
+	# Check to see if this is a sensor (forward) node
+	is_single_node_grid && return 0
+	grep "role: so-" /etc/salt/grains | grep -E "sensor|heavynode|helix" &> /dev/null
+}
+
+is_single_node_grid() {
+	grep "role: so-" /etc/salt/grains | grep -E "eval|standalone|import" &> /dev/null
+}
+
+lookup_bond_interfaces() {
+	cat /proc/net/bonding/bond0 | grep "Slave Interface:" | sed -e "s/Slave Interface: //g"
+}
+
+lookup_salt_value() {
+	key=$1
+	group=$2
+	kind=$3
+	output=${4:-newline_values_only}
+	local=$5
+
+	if [ -z "$kind" ]; then
+		kind=pillar
+	fi
+
+	if [ -n "$group" ]; then
+		group=${group}:
+	fi
+
+	if [[ "$local" == "--local" ]] || [[ "$local" == "local" ]]; then
+		local="--local"
+	else
+		local=""
+	fi
+
+	salt-call --no-color  ${kind}.get ${group}${key} --out=${output} ${local}
+}
+
+lookup_pillar() {
+	key=$1
+	pillar=$2
+	if [ -z "$pillar" ]; then
+		pillar=global
+	fi
+	lookup_salt_value "$key" "$pillar" "pillar"
+}
+
+lookup_pillar_secret() {
+	lookup_pillar "$1" "secrets"
+}
+
+lookup_grain() {
+	lookup_salt_value "$1" "" "grains"
+}
+
+lookup_role() {
+	id=$(lookup_grain id)
+	pieces=($(echo $id | tr '_' ' '))
+	echo ${pieces[1]}
+}
+
+require_manager() {
+	if is_manager_node; then
+		echo "This is a manager, so we can proceed."
+	else
+		echo "Please run this command on the manager; the manager controls the grid."
+		exit 1
+	fi
+}
+
+retry() {
+        maxAttempts=$1
+        sleepDelay=$2
+        cmd=$3
+        expectedOutput=$4
+        failedOutput=$5
+        attempt=0
+        local exitcode=0
+        while [[ $attempt -lt $maxAttempts ]]; do
+                attempt=$((attempt+1))
+                echo "Executing command with retry support: $cmd"
+                output=$(eval "$cmd")
+                exitcode=$?
+                echo "Results: $output ($exitcode)"
+                if [ -n "$expectedOutput" ]; then
+                        if [[ "$output" =~ "$expectedOutput" ]]; then
+                                return $exitcode
+                        else
+                                echo "Did not find expectedOutput: '$expectedOutput' in the output below from running the command: '$cmd'"
+								echo "<Start of output>"
+								echo "$output"
+								echo "<End of output>"
+                        fi
+                elif [ -n "$failedOutput" ]; then
+                        if [[ "$output" =~ "$failedOutput" ]]; then
+                                echo "Found failedOutput: '$failedOutput' in the output below from running the command: '$cmd'"
+								echo "<Start of output>"
+								echo "$output"
+								echo "<End of output>"
+								if [[ $exitcode -eq 0 ]]; then
+									echo "The exitcode was 0, but we are setting to 1 since we found $failedOutput in the output."
+									exitcode=1
+								fi
+                        else
+                                return $exitcode
+                        fi
+                elif [[ $exitcode -eq 0 ]]; then
+                        return $exitcode
+                fi
+                echo "Command failed with exit code $exitcode; will retry in $sleepDelay seconds ($attempt / $maxAttempts)..."
+                sleep $sleepDelay
+        done
+        echo "Command continues to fail; giving up."
+        return $exitcode
+}
+
+run_check_net_err() {
+	local cmd=$1
+	local err_msg=${2:-"Unknown error occured, please check /root/$WHATWOULDYOUSAYYAHDOHERE.log for details."} # Really need to rename that variable
+	local no_retry=$3
+
+	local exit_code
+	if [[ -z $no_retry ]]; then
+		retry 5 60 "$cmd"
+		exit_code=$?
+	else
+		eval "$cmd"
+		exit_code=$?
+	fi
+	
+	if [[ $exit_code -ne 0 ]]; then
+		ERR_HANDLED=true
+		[[ -z $no_retry ]] || echo "Command failed with error $exit_code"
+		echo "$err_msg"
+		exit $exit_code
+	fi
+}
+
+salt_minion_count() {
+	local MINIONDIR="/opt/so/saltstack/local/pillar/minions"
+	MINIONCOUNT=$(ls -la $MINIONDIR/*.sls | grep -v adv_ | wc -l)
+
+}
+
+set_cron_service_name() {
+	if [[ "$OS" == "rocky" ]]; then
+		cron_service_name="crond"
+	else
+		cron_service_name="cron"
+	fi
+}
+
+set_os() {
+	if [ -f /etc/redhat-release ]; then
+		OS=rocky
+	else
+		OS=ubuntu
+	fi
+}
+
+set_minionid() {
+	MINIONID=$(lookup_grain id)
+}
+
+set_palette() {
+	if [ "$OS" == ubuntu ]; then
+	    update-alternatives --set newt-palette /etc/newt/palette.original
+    fi
+}
+
+set_version() {
+	CURRENTVERSION=0.0.0
+	if [ -f /etc/soversion ]; then
+		CURRENTVERSION=$(cat /etc/soversion)
+	fi
+	if [ -z "$VERSION" ]; then
+		if [ -z "$NEWVERSION" ]; then
+			if [ "$CURRENTVERSION" == "0.0.0" ]; then
+				echo "ERROR: Unable to detect Security Onion version; terminating script."
+				exit 1
+			else
+				VERSION=$CURRENTVERSION
+			fi
+		else
+			VERSION="$NEWVERSION"
+		fi
+	fi
+}
+
+systemctl_func() {
+	local action=$1
+	local echo_action=$1
+	local service_name=$2
+
+	if [[ "$echo_action" == "stop" ]]; then
+		echo_action="stopp"
+	fi
+
+	echo ""
+	echo "${echo_action^}ing $service_name service at $(date +"%T.%6N")"
+    systemctl $action $service_name && echo "Successfully ${echo_action}ed $service_name." || echo "Failed to $action $service_name."
+	echo ""
+}
+
+has_uppercase() {
+	local string=$1
+
+	echo "$string" | grep -qP '[A-Z]' \
+		&& return 0 \
+		|| return 1
+}
+
+valid_cidr() {
+	# Verify there is a backslash in the string
+	echo "$1" | grep -qP "^[^/]+/[^/]+$" || return 1
+
+	valid_ip4_cidr_mask "$1" && return 0 || return 1
+	
+	local cidr="$1"
+	local ip
+	ip=$(echo "$cidr" | sed 's/\/.*//' )
+	
+	if valid_ip4 "$ip"; then
+		local ip1 ip2 ip3 ip4 N
+		IFS="./" read -r ip1 ip2 ip3 ip4 N <<< "$cidr"
+		ip_total=$((ip1 * 256 ** 3 + ip2 * 256 ** 2 + ip3 * 256 + ip4))
+		[[ $((ip_total % 2**(32-N))) == 0 ]] && return 0 || return 1
+	else
+		return 1
+	fi
+}
+
+valid_cidr_list() {
+	local all_valid=0
+
+	IFS="," read -r -a net_arr <<< "$1"
+
+	for net in "${net_arr[@]}"; do
+		valid_cidr "$net" || all_valid=1
+	done
+
+	return $all_valid
+}
+
+valid_dns_list() {
+	local all_valid=0
+
+	IFS="," read -r -a dns_arr <<< "$1"
+
+	for addr in "${dns_arr[@]}"; do
+		valid_ip4 "$addr" || all_valid=1
+	done
+
+	return $all_valid
+}
+
+valid_fqdn() {
+	local fqdn=$1
+
+	echo "$fqdn" | grep -qP '(?=^.{4,253}$)(^((?!-)[a-zA-Z0-9-]{0,62}[a-zA-Z0-9]\.)+[a-zA-Z]{2,63}$)' \
+		&& return 0 \
+		|| return 1
+}
+
+valid_hostname() {
+	local hostname=$1
+
+	[[ $hostname =~ ^[a-zA-Z0-9\-]+$ ]] && [[ $hostname != 'localhost' ]] && return 0 || return 1
+}
+
+verify_ip4() {
+        local ip=$1
+        # Is this an IP or CIDR?
+        if grep -qP "^[^/]+/[^/]+$" <<< $ip; then
+                # Looks like a CIDR
+                valid_ip4_cidr_mask "$ip"
+        else
+                # We know this is not a CIDR - Is it an IP?
+                valid_ip4 "$ip"
+        fi
+}
+
+valid_ip4() {
+	local ip=$1
+
+	echo "$ip" | grep -qP '^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$' && return 0 || return 1
+}
+
+valid_ip4_cidr_mask() {
+	# Verify there is a backslash in the string
+	echo "$1" | grep -qP "^[^/]+/[^/]+$" || return 1
+
+	local cidr
+	local ip
+
+	cidr=$(echo "$1" | sed 's/.*\///')
+	ip=$(echo "$1" | sed 's/\/.*//' )
+
+	if valid_ip4 "$ip"; then
+		[[ $cidr =~ ^([0-9]|[1-2][0-9]|3[0-2])$ ]] && return 0 || return 1
+	else
+		return 1
+	fi
+}
+
+valid_int() {
+	local num=$1
+	local min=${2:-1}
+	local max=${3:-1000000000}
+
+	[[ $num =~ ^[0-9]*$ ]] && [[ $num -ge $min ]] && [[ $num -le $max ]] && return 0 || return 1
+}
+
+# {% raw %}
+
+valid_proxy() {
+	local proxy=$1
+	local url_prefixes=( 'http://' 'https://' )
+
+	local has_prefix=false
+	for prefix in "${url_prefixes[@]}"; do
+		echo "$proxy" | grep -q "$prefix" && has_prefix=true && proxy=${proxy#"$prefix"} && break
+	done
+	
+	local url_arr
+	mapfile -t url_arr <<< "$(echo "$proxy" | tr ":" "\n")"
+
+	local valid_url=true
+	if ! valid_ip4 "${url_arr[0]}" && ! valid_fqdn "${url_arr[0]}" && ! valid_hostname "${url_arr[0]}"; then
+		valid_url=false
+	fi
+
+	[[ $has_prefix == true ]] && [[ $valid_url == true ]] && return 0 || return 1
+}
+
+valid_ntp_list() {
+	local string=$1
+	local ntp_arr
+	IFS="," read -r -a ntp_arr <<< "$string"
+
+	for ntp in "${ntp_arr[@]}"; do
+		if ! valid_ip4 "$ntp" && ! valid_hostname "$ntp" && ! valid_fqdn "$ntp"; then
+			return 1
+		fi
+	done
+
+	return 0
+}
+
+valid_string() {
+	local str=$1
+	local min_length=${2:-1}
+	local max_length=${3:-64}
+
+	echo "$str" | grep -qP '^\S+$' && [[ ${#str} -ge $min_length ]] && [[ ${#str} -le $max_length ]] && return 0 || return 1
+}
+
+# {% endraw %}
+
+valid_username() {
+	local user=$1
+
+	echo "$user" | grep -qP '^[a-z_]([a-z0-9_-]{0,31}|[a-z0-9_-]{0,30}\$)$' && return 0 || return 1
+}
+
+wait_for_web_response() {
+	url=$1
+	expected=$2
+	maxAttempts=${3:-300}
+	curlcmd=${4:-curl}
+	logfile=/root/wait_for_web_response.log
+	truncate -s 0 "$logfile"
+	attempt=0
+	while [[ $attempt -lt $maxAttempts ]]; do
+		attempt=$((attempt+1))
+		echo "Waiting for value '$expected' at '$url' ($attempt/$maxAttempts)"
+		result=$($curlcmd -ks -L $url)
+		exitcode=$?
+
+		echo "--------------------------------------------------" >> $logfile
+		echo "$(date) - Checking web URL: $url ($attempt/$maxAttempts)" >> $logfile
+		echo "$result" >> $logfile
+		echo "exit code=$exitcode" >> $logfile
+		echo "" >> $logfile
+
+		if [[ $exitcode -eq 0 && "$result" =~ $expected ]]; then
+			echo "Received expected response; proceeding."
+			return 0
+		fi
+		echo "Server is not ready"
+		sleep 1
+	done
+	echo "Server still not ready after $maxAttempts attempts; giving up."
+	return 1
+}

salt/common/tools/sbin/so-config-backup

@@ -1,44 +0,0 @@
-#!/bin/bash
-#
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.. /usr/sbin/so-common
-{% set BACKUPLOCATIONS = salt['pillar.get']('backup:locations', {}) %}
-
-TODAY=$(date '+%Y_%m_%d')
-BACKUPFILE="/nsm/backup/so-config-backup-$TODAY.tar"
-MAXBACKUPS=7
-
-# Create backup dir if it does not exist
-mkdir -p /nsm/backup
-
-# If we haven't already written a backup file for today, let's do so
-if [ ! -f $BACKUPFILE ]; then
-
-  # Create empty backup file
-  tar -cf $BACKUPFILE -T /dev/null
-
-  # Loop through all paths defined in global.sls, and append them to backup file
-  {%- for LOCATION in BACKUPLOCATIONS %}
-  tar -rf $BACKUPFILE {{ LOCATION }}
-  {%- endfor %}
-
-fi
-
-# Find oldest backup file and remove it
-NUMBACKUPS=$(find /nsm/backup/ -type f -name "so-config-backup*" | wc -l)
-OLDESTBACKUP=$(find /nsm/backup/ -type f -name "so-config-backup*" | ls -1t | tail -1)
-if [ "$NUMBACKUPS" -gt "$MAXBACKUPS" ]; then
-  rm -f /nsm/backup/$OLDESTBACKUP
-fi

salt/common/tools/sbin/so-cortex-restart

@@ -1,21 +0,0 @@
-#!/bin/bash
-#
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-. /usr/sbin/so-common
-
-/usr/sbin/so-stop cortex $1
-/usr/sbin/so-start thehive $1

salt/common/tools/sbin/so-cortex-start

@@ -1,20 +0,0 @@
-#!/bin/bash
-
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-. /usr/sbin/so-common
-
-/usr/sbin/so-start thehive $1

salt/common/tools/sbin/so-cortex-stop

@@ -1,20 +0,0 @@
-#!/bin/bash
-#
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-. /usr/sbin/so-common
-
-/usr/sbin/so-stop cortex $1

salt/common/tools/sbin/so-cortex-user-add

@@ -1,54 +0,0 @@
-#!/bin/bash
-#
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-. /usr/sbin/so-common
-
-usage() {
-    echo "Usage: $0 <new-user-name>"
-    echo ""
-    echo "Adds a new user to Cortex. The new password will be read from STDIN."
-    exit 1
-}
-
-if [ $# -ne 1 ]; then
-  usage
-fi
-
-USER=$1
-
-CORTEX_KEY=$(lookup_pillar cortexkey)
-CORTEX_API_URL="$(lookup_pillar url_base)/cortex/api"
-CORTEX_ORG_NAME=$(lookup_pillar cortexorgname)
-CORTEX_USER=$USER
-
-# Read password for new user from stdin
-test -t 0
-if [[ $? == 0 ]]; then
-  echo "Enter new password:"
-fi
-read -rs CORTEX_PASS
-
-# Create new user in Cortex
-resp=$(curl -sk -XPOST -H "Authorization: Bearer $CORTEX_KEY" -H "Content-Type: application/json" -L "https://$CORTEX_API_URL/user" -d "{\"name\": \"$CORTEX_USER\",\"roles\": [\"read\",\"analyze\",\"orgadmin\"],\"organization\": \"$CORTEX_ORG_NAME\",\"login\": \"$CORTEX_USER\",\"password\" : \"$CORTEX_PASS\" }")
-if [[ "$resp" =~ \"status\":\"Ok\" ]]; then
-    echo "Successfully added user to Cortex."
-else
-    echo "Unable to add user to Cortex; user might already exist."
-    echo $resp
-    exit 2
-fi
-    

salt/common/tools/sbin/so-cortex-user-enable

@@ -1,57 +0,0 @@
-#!/bin/bash
-#
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-. /usr/sbin/so-common
-
-usage() {
-    echo "Usage: $0 <user-name> <true|false>"
-    echo ""
-    echo "Enables or disables a user in Cortex."
-    exit 1
-}
-
-if [ $# -ne 2 ]; then
-  usage
-fi
-
-USER=$1
-
-CORTEX_KEY=$(lookup_pillar cortexkey)
-CORTEX_API_URL="$(lookup_pillar url_base)/cortex/api"
-CORTEX_USER=$USER
-
-case "${2^^}" in
-	FALSE | NO | 0)
-		CORTEX_STATUS=Locked
-		;;
-	TRUE | YES | 1)
-		CORTEX_STATUS=Ok
-		;;
-	*)
-		usage
-		;;
-esac
-
-resp=$(curl -sk -XPATCH -H "Authorization: Bearer $CORTEX_KEY" -H "Content-Type: application/json" -L "https://$CORTEX_API_URL/user/${CORTEX_USER}" -d "{\"status\":\"${CORTEX_STATUS}\" }")
-if [[ "$resp" =~ \"status\":\"Locked\" || "$resp" =~ \"status\":\"Ok\" ]]; then
-    echo "Successfully updated user in Cortex."
-else
-    echo "Failed to update user in Cortex."
-    echo $resp
-    exit 2
-fi
-    

salt/common/tools/sbin/so-curator-restart

@@ -1,19 +1,11 @@
 #!/bin/bash
 
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+
 
 . /usr/sbin/so-common
 

salt/common/tools/sbin/so-curator-start

@@ -1,19 +1,11 @@
 #!/bin/bash
 
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+
 
 . /usr/sbin/so-common
 

salt/common/tools/sbin/so-curator-stop

@@ -1,19 +1,11 @@
 #!/bin/bash
 
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+
 
 . /usr/sbin/so-common
 

salt/common/tools/sbin/so-deny

@@ -0,0 +1,138 @@
+#!/usr/bin/env python3
+
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+
+
+import ipaddress
+import textwrap
+import os
+import subprocess
+import sys
+import argparse
+import re
+from lxml import etree as ET
+from xml.dom import minidom
+
+
+LOCAL_SALT_DIR='/opt/so/saltstack/local'
+VALID_ROLES = {
+  'a': { 'role': 'analyst','desc': 'Analyst - 80/tcp, 443/tcp' },
+  'b': { 'role': 'beats_endpoint', 'desc': 'Logstash Beat - 5044/tcp' },
+  'e': { 'role': 'elasticsearch_rest', 'desc': 'Elasticsearch REST API - 9200/tcp' },
+  'f': { 'role': 'strelka_frontend', 'desc': 'Strelka frontend - 57314/tcp' },
+  's': { 'role': 'syslog', 'desc': 'Syslog device - 514/tcp/udp' },
+}
+
+
+def validate_ip_cidr(ip_cidr: str) -> bool:
+  try:
+    ipaddress.ip_address(ip_cidr)
+  except ValueError:
+    try:
+      ipaddress.ip_network(ip_cidr)
+    except ValueError:
+      return False
+  return True
+
+
+def role_prompt() -> str:
+  print()
+  print('Choose the role for the IP or Range you would like to deny')
+  print()
+  for role in VALID_ROLES:
+    print(f'[{role}] - {VALID_ROLES[role]["desc"]}')
+  print()
+  role = input('Please enter your selection: ')
+  if role in VALID_ROLES.keys():
+    return VALID_ROLES[role]['role']
+  else:
+    print(f'Invalid role \'{role}\', please try again.', file=sys.stderr)
+    sys.exit(1)
+  
+
+def ip_prompt() -> str:
+  ip = input('Enter a single ip address or range to deny (ex: 10.10.10.10 or 10.10.0.0/16): ')
+  if validate_ip_cidr(ip):
+    return ip
+  else:
+    print(f'Invalid IP address or CIDR block \'{ip}\', please try again.', file=sys.stderr)
+    sys.exit(1)
+
+
+def apply(role: str, ip: str) -> int:
+  firewall_cmd = ['so-firewall', 'excludehost', role, ip]
+  salt_cmd = ['salt-call', 'state.apply', '-l', 'quiet', 'firewall', 'queue=True']
+  print(f'Removing {ip} from the {role} role. This can take a few seconds...')
+  cmd = subprocess.run(firewall_cmd)
+  if cmd.returncode == 0:
+    cmd = subprocess.run(salt_cmd, stdout=subprocess.DEVNULL)
+  else:
+    return cmd.returncode
+
+
+def main():
+  if os.geteuid() != 0:
+    print('You must run this script as root', file=sys.stderr)
+    sys.exit(1)
+
+  main_parser = argparse.ArgumentParser(
+    formatter_class=argparse.RawDescriptionHelpFormatter,
+    epilog=textwrap.dedent(f'''\
+        additional information:
+                      To use this script in interactive mode call it with no arguments
+        '''
+  ))
+
+  group = main_parser.add_argument_group(title='roles')
+  group.add_argument('-a', dest='roles', action='append_const', const=VALID_ROLES['a']['role'], help="Analyst - 80/tcp, 443/tcp")
+  group.add_argument('-b', dest='roles', action='append_const', const=VALID_ROLES['b']['role'], help="Logstash Beat - 5044/tcp")
+  group.add_argument('-e', dest='roles', action='append_const', const=VALID_ROLES['e']['role'], help="Elasticsearch REST API - 9200/tcp")
+  group.add_argument('-f', dest='roles', action='append_const', const=VALID_ROLES['f']['role'], help="Strelka frontend - 57314/tcp")
+  group.add_argument('-s', dest='roles', action='append_const', const=VALID_ROLES['s']['role'], help="Syslog device - 514/tcp/udp")
+  
+  ip_g = main_parser.add_argument_group(title='allow')
+  ip_g.add_argument('-i', help="IP or CIDR block to disallow connections from, requires at least one role argument", metavar='', dest='ip')
+
+  args = main_parser.parse_args(sys.argv[1:])
+
+  if args.roles is None:
+    role = role_prompt()
+    ip = ip_prompt()
+    try:
+      return_code = apply(role, ip)
+    except Exception as e:
+      print(f'Unexpected exception occurred: {e}', file=sys.stderr)
+      return_code = e.errno
+    sys.exit(return_code)
+  elif args.roles is not None and args.ip is None:
+    if os.environ.get('IP') is None:
+      main_parser.print_help()
+      sys.exit(1)
+    else:
+      args.ip = os.environ['IP']
+  
+  if validate_ip_cidr(args.ip):
+    try:
+      for role in args.roles:
+        return_code = apply(role, args.ip)
+        if return_code > 0:
+          break
+    except Exception as e:
+      print(f'Unexpected exception occurred: {e}', file=sys.stderr)
+      return_code = e.errno
+  else:
+    print(f'Invalid IP address or CIDR block \'{args.ip}\', please try again.', file=sys.stderr)
+    return_code = 1
+    
+  sys.exit(return_code)
+
+
+if __name__ == '__main__':
+  try:
+    main()
+  except KeyboardInterrupt:
+    sys.exit(1)

salt/common/tools/sbin/so-docker-prune

@@ -0,0 +1,94 @@
+#!/usr/bin/env python3
+
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+
+
+import sys, argparse, re, docker
+from packaging.version import Version, InvalidVersion
+from itertools import groupby, chain
+
+
+def get_image_name(string) -> str:
+  return ':'.join(string.split(':')[:-1])
+
+
+def get_so_image_basename(string) -> str:
+  return get_image_name(string).split('/so-')[-1]
+
+
+def get_image_version(string) -> str:
+  ver = string.split(':')[-1]
+  if ver == 'latest':
+    # Version doesn't like "latest", so use a high semver
+    return '99999.9.9'
+  else:
+    try:
+      Version(ver)
+    except InvalidVersion:
+      # Also return a very high semver for any version 
+      # with a dash in it since it will likely be a dev version of some kind
+      if '-' in ver:
+        return '999999.9.9'
+    return ver
+
+
+def main(quiet):
+  client = docker.from_env()
+
+  # Prune old/stopped containers
+  if not quiet: print('Pruning old containers')
+  client.containers.prune()
+
+  image_list = client.images.list(filters={ 'dangling': False })
+
+  # Map list of image objects to flattened list of tags (format: "name:version")
+  tag_list = list(chain.from_iterable(list(map(lambda x: x.attrs.get('RepoTags'), image_list))))
+
+  # Filter to only SO images (base name begins with "so-")
+  tag_list = list(filter(lambda x: re.match(r'^.*\/so-[^\/]*$', get_image_name(x)), tag_list))
+
+  # Group tags into lists by base name (sort by same projection first)
+  tag_list.sort(key=lambda x: get_so_image_basename(x))
+  grouped_tag_lists = [ list(it) for _, it in groupby(tag_list, lambda x: get_so_image_basename(x)) ]
+
+  no_prunable = True
+  for t_list in grouped_tag_lists:
+    try:
+      # Group tags by version, in case multiple images exist with the same version string
+      t_list.sort(key=lambda x: Version(get_image_version(x)), reverse=True)
+      grouped_t_list = [ list(it) for _,it in groupby(t_list, lambda x: get_image_version(x)) ]
+
+      # Keep the 2 most current version groups
+      if len(grouped_t_list) <= 2:
+        continue
+      else:
+        no_prunable = False
+        for group in grouped_t_list[2:]:
+          for tag in group:
+            if not quiet: print(f'Removing image {tag}')
+            try:
+              client.images.remove(tag, force=True)
+            except docker.errors.ClientError as e:
+              print(f'Could not remove image {tag}, continuing...')
+    except (docker.errors.APIError, InvalidVersion) as e:
+      print(f'so-{get_so_image_basename(t_list[0])}: {e}', file=sys.stderr)
+      exit(1)
+    except Exception as e:
+      print('Unhandled exception occurred:')
+      print(f'so-{get_so_image_basename(t_list[0])}: {e}', file=sys.stderr)
+      exit(1)
+
+  if no_prunable and not quiet:
+    print('No Security Onion images to prune')
+
+
+if __name__ == "__main__":
+  main_parser = argparse.ArgumentParser(add_help=False)
+  main_parser.add_argument('-q', '--quiet', action='store_const', const=True, required=False)
+  args = main_parser.parse_args(sys.argv[1:])
+ 
+  main(args.quiet)

salt/common/tools/sbin/so-docker-refresh

@@ -1,19 +1,11 @@
 #!/bin/bash
 
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+
 
 . /usr/sbin/so-common
 . /usr/sbin/so-image-common

salt/common/tools/sbin/so-elastalert-create

@@ -145,9 +145,9 @@ EOF
    rulename=$(echo ${raw_rulename,,} | sed 's/ /_/g')
 
 cat << EOF >> "$rulename.yaml"
-# Elasticsearch Host
-es_host: elasticsearch
-es_port: 9200
+# Elasticsearch Host Override (optional)
+# es_host: elasticsearch
+# es_port: 9200
 
 # (Required)
 # Rule name, must be unique

salt/common/tools/sbin/so-elastalert-restart

@@ -1,19 +1,11 @@
 #!/bin/bash
 
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+
 
 . /usr/sbin/so-common
 

salt/common/tools/sbin/so-elastalert-start

@@ -1,19 +1,11 @@
 #!/bin/bash
 
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+
 
 . /usr/sbin/so-common
 

salt/common/tools/sbin/so-elastalert-stop

@@ -1,19 +1,11 @@
 #!/bin/bash
 
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+
 
 . /usr/sbin/so-common
 

salt/common/tools/sbin/so-elastalert-test

@@ -70,7 +70,7 @@ do
 done
 
 docker_exec(){
-	CMD="docker exec -it so-elastalert elastalert-test-rule /opt/elastalert/rules/$RULE_NAME --config /opt/config/elastalert_config.yaml $OPTIONS"
+	CMD="docker exec -it so-elastalert elastalert-test-rule /opt/elastalert/rules/$RULE_NAME --config /opt/elastalert/config.yaml $OPTIONS"
 	if [ "${RESULTS_TO_LOG,,}" = "y" ] ; then
 		$CMD > "$FILE_SAVE_LOCATION"
 	else
@@ -96,7 +96,7 @@ rule_prompt(){
 	echo "-----------------------------------"
 	echo
 	while [ -z "$RULE_NAME" ]; do
-		read -p "Please enter the rule filename you want to test (filename only, no path): " -e RULE_NAME
+		read -p "Choose a rule to test from the list above (must be typed exactly as shown above): " -e RULE_NAME
 	done
 }
 

salt/common/tools/sbin/so-elastic-agent-gen-installers

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use
+# this file except in compliance with the Elastic License 2.0.
+
+#so-elastic-agent-gen-installers $FleetHost $EnrollmentToken 
+
+{% from 'vars/globals.map.jinja' import GLOBALS %}
+
+. /usr/sbin/so-common
+
+ENROLLMENTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'  | jq .list | jq -r -c '.[] | select(.policy_id | contains("endpoints")) | .api_key')
+
+#FLEETHOST=$(lookup_pillar "server:url" "elasticfleet")
+FLEETHOST="{{ GLOBALS.manager_ip }}"
+
+#FLEETHOST=$1
+#ENROLLMENTOKEN=$2
+CONTAINERGOOS=( "linux" "darwin" "windows" )
+
+#rm -rf /tmp/elastic-agent-workspace
+#mkdir -p /tmp/elastic-agent-workspace
+
+for OS in "${CONTAINERGOOS[@]}"
+do
+    printf "\n\nGenerating $OS Installer..."
+    #cp /opt/so/saltstack/default/salt/elasticfleet/files/elastic-agent/so-elastic-agent-*-$OS-x86_64.tar.gz /tmp/elastic-agent-workspace/$OS.tar.gz
+    docker run -e CGO_ENABLED=0 -e GOOS=$OS \
+    --mount type=bind,source=/etc/ssl/certs/,target=/workspace/files/cert/ \
+    --mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/,target=/output/ \
+    {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} go build -ldflags "-X main.fleetHost=$FLEETHOST -X main.enrollmentToken=$ENROLLMENTOKEN"  -o /output/so-elastic-agent_$OS
+    printf "\n $OS Installer Generated..."
+done

salt/common/tools/sbin/so-elastic-auth-password-reset

@@ -0,0 +1,144 @@
+#!/bin/bash
+
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+
+source $(dirname $0)/so-common
+require_manager
+
+user=$1
+elasticUsersFile=${ELASTIC_USERS_FILE:-/opt/so/saltstack/local/salt/elasticsearch/files/users}
+elasticAuthPillarFile=${ELASTIC_AUTH_PILLAR_FILE:-/opt/so/saltstack/local/pillar/elasticsearch/auth.sls}
+
+if [[ $# -ne 1 ]]; then
+  echo "Usage: $0 <user>"
+  echo ""
+  echo " where <user> is one of the following:"
+  echo ""
+  echo "         all: Reset the password for the so_elastic, so_kibana, so_logstash, so_beats, and so_monitor users"
+  echo "  so_elastic: Reset the password for the so_elastic user"
+  echo "   so_kibana: Reset the password for the so_kibana user"
+  echo " so_logstash: Reset the password for the so_logstash user"
+  echo "    so_beats: Reset the password for the so_beats user"
+  echo "  so_monitor: Reset the password for the so_monitor user"
+  echo ""
+  exit 1
+fi
+
+# function to create a lock so that the so-user sync cronjob can't run while this is running
+function lock() {
+  # Obtain file descriptor lock
+  exec 99>/var/tmp/so-user.lock || fail "Unable to create lock descriptor; if the system was not shutdown gracefully you may need to remove /var/tmp/so-user.lock manually."
+  flock -w 10 99 || fail "Another process is using so-user; if the system was not shutdown gracefully you may need to remove /var/tmp/so-user.lock manually."
+  trap 'rm -f /var/tmp/so-user.lock' EXIT
+}
+
+function unlock() {
+  rm -f /var/tmp/so-user.lock
+}
+
+function fail() {
+  msg=$1
+  echo "$1"
+  exit 1
+}
+
+function removeSingleUserPass() {
+  local user=$1
+  sed -i '/user: '"${user}"'/{N;/pass: /d}' "${elasticAuthPillarFile}"
+}
+
+function removeAllUserPass() {
+  local userList=("so_elastic" "so_kibana" "so_logstash" "so_beats" "so_monitor")
+
+  for u in ${userList[@]}; do
+    removeSingleUserPass "$u"
+  done
+}
+
+function removeElasticUsersFile() {
+  rm -f "$elasticUsersFile"
+}
+
+function createElasticAuthPillar() {
+  salt-call state.apply elasticsearch.auth queue=True
+}
+
+# this will disable highstate to prevent a highstate from starting while the script is running
+# will also disable salt.minion-state-apply-test allow so-salt-minion-check cronjob to restart salt-minion service incase
+function disableSaltStates() {
+  printf "\nDisabling salt.minion-state-apply-test and highstate from running.\n\n"
+  salt-call state.disable salt.minion-state-apply-test
+  salt-call state.disable highstate
+}
+
+function enableSaltStates() {
+  printf "\nEnabling salt.minion-state-apply-test and highstate.\n\n"
+  salt-call state.enable salt.minion-state-apply-test
+  salt-call state.enable highstate
+}
+
+function killAllSaltJobs() {
+  printf "\nKilling all running salt jobs.\n\n"
+  salt-call saltutil.kill_all_jobs
+}
+
+function soUserSync() {
+  # apply this state to update /opt/so/saltstack/local/salt/elasticsearch/curl.config on the manager
+  salt-call state.sls_id elastic_curl_config_distributed manager queue=True
+  salt -C 'G@role:so-standalone or G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managersearch or G@role:so-searchnode or G@role:so-heavynode' saltutil.kill_all_jobs
+  # apply this state to get the curl.config
+  salt -C 'G@role:so-standalone or G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managersearch or G@role:so-searchnode or G@role:so-heavynode' state.sls_id elastic_curl_config common queue=True
+  $(dirname $0)/so-user sync
+  printf "\nApplying logstash state to the appropriate nodes.\n\n"
+  salt -C 'G@role:so-standalone or G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managersearch or G@role:so-searchnode or G@role:so-heavynode' state.apply logstash queue=True
+  printf "\nApplying kibana state to the appropriate nodes.\n\n"
+  salt -C 'G@role:so-standalone or G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managersearch' state.apply kibana queue=True
+  printf "\nApplying curator state to the appropriate nodes.\n\n"
+  salt -C 'G@role:so-standalone or G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managersearch or G@role:so-searchnode or G@role:so-heavynode' state.apply curator queue=True
+}
+
+function highstateManager() {
+  killAllSaltJobs
+  printf "\nRunning highstate on the manager to finalize password reset.\n\n"
+  salt-call state.highstate -linfo queue=True
+}
+
+case "${user}" in
+
+  so_elastic | so_kibana | so_logstash | so_beats | so_monitor)
+    lock
+    killAllSaltJobs
+    disableSaltStates
+    removeSingleUserPass "$user"
+    createElasticAuthPillar
+    removeElasticUsersFile
+    unlock
+    soUserSync
+    enableSaltStates
+    highstateManager
+    ;;
+
+  all)
+    lock
+    killAllSaltJobs
+    disableSaltStates
+    removeAllUserPass
+    createElasticAuthPillar
+    removeElasticUsersFile
+    unlock
+    soUserSync
+    enableSaltStates
+    highstateManager
+    ;;
+
+  *)
+    fail "Unsupported user: $user"
+    ;;
+
+esac
+
+exit 0

salt/common/tools/sbin/so-elastic-clear

@@ -1,20 +1,11 @@
 #!/bin/bash
 #
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-{%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%}
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{%- set NODEIP = salt['pillar.get']('host:mainip', '') %}
 . /usr/sbin/so-common
 
 SKIP=0
@@ -30,16 +21,34 @@ Security Onion Elastic Clear
   -y         Skip interactive mode
 EOF
 }
-while getopts "h:y" OPTION
+while getopts "h:cdely" OPTION
 do
         case $OPTION in
                 h)
                         usage
                         exit 0
                         ;;
-                
-                y)
+                c)
+			DELETE_CASES_DATA=1
+			SKIP=1
+                        ;;
+                d)
+			DONT_STOP_SERVICES=1
                         SKIP=1
+			;;
+                e)
+			DELETE_ELASTALERT_DATA=1
+                        SKIP=1
+                        ;;
+		l)
+			DELETE_LOG_DATA=1
+			SKIP=1
+			;;
+                y)
+                        DELETE_CASES_DATA=1
+                        DELETE_ELASTALERT_DATA=1
+			DELETE_LOG_DATA=1
+			SKIP=1
                         ;;
                 *)
                         usage
@@ -50,11 +59,7 @@ done
 if [ $SKIP -ne 1 ]; then
         # List indices
         echo 
-        {% if grains['role'] in ['so-node','so-heavynode'] %}
-        curl -k -L https://{{ NODEIP }}:9200/_cat/indices?v
-        {% else %}
-        curl -L {{ NODEIP }}:9200/_cat/indices?v
-        {% endif %}
+        curl -K /opt/so/conf/elasticsearch/curl.config -k -L https://{{ NODEIP }}:9200/_cat/indices?v
         echo
         # Inform user we are about to delete all data
         echo
@@ -67,62 +72,83 @@ if [ $SKIP -ne 1 ]; then
         if [ "$INPUT" != "AGREE" ] ; then exit 0; fi
 fi
 
-# Check to see if Logstash/Filebeat are running
-LS_ENABLED=$(so-status | grep logstash)
-FB_ENABLED=$(so-status | grep filebeat)
-EA_ENABLED=$(so-status | grep elastalert)
 
-if [ ! -z "$FB_ENABLED" ]; then
+if [ -z "$DONT_STOP_SERVICES" ]; then
+	# Stop Elastic Agent
+	for i in $(pgrep elastic-agent | grep -v grep); do
+		kill -9 $i;
+	done 
 
-        /usr/sbin/so-filebeat-stop
+	# Check to see if Elastic Fleet, Logstash, Elastalert are running
+	#EF_ENABLED=$(so-status | grep elastic-fleet)
+	LS_ENABLED=$(so-status | grep logstash)
+	EA_ENABLED=$(so-status | grep elastalert)
 
+	#if [ ! -z "$EF_ENABLED" ]; then
+        #	/usr/sbin/so-elastic-fleet-stop
+	#fi
+
+	if [ ! -z "$LS_ENABLED" ]; then
+        	/usr/sbin/so-logstash-stop
+	fi
+
+	if [ ! -z "$EA_ENABLED" ]; then
+        	/usr/sbin/so-elastalert-stop
+	fi
 fi
 
-if [ ! -z "$LS_ENABLED" ]; then
-
-        /usr/sbin/so-logstash-stop
-
+if [ ! -z "$DELETE_CASES_DATA" ]; then
+        # Delete Cases data
+        echo "Deleting Cases data..."
+        INDXS=$(/usr/sbin/so-elasticsearch-query _cat/indices?h=index | grep "so-case")
+        for INDX in ${INDXS}
+        do
+                echo "Deleting $INDX"
+                /usr/sbin/so-elasticsearch-query ${INDX} -XDELETE > /dev/null 2>&1
+        done
 fi
 
-if [ ! -z "$EA_ENABLED" ]; then
-
-        /usr/sbin/so-elastalert-stop
-
+# Delete Elastalert data
+if [ ! -z "$DELETE_ELASTALERT_DATA" ]; then
+        # Delete Elastalert data
+        echo "Deleting Elastalert data..."
+        INDXS=$(/usr/sbin/so-elasticsearch-query _cat/indices?h=index | grep "elastalert")
+        for INDX in ${INDXS}
+        do
+                echo "Deleting $INDX"
+                /usr/sbin/so-elasticsearch-query ${INDX} -XDELETE > /dev/null 2>&1
+        done
 fi
 
-# Delete data
-echo "Deleting data..."
-
-{% if grains['role'] in ['so-node','so-heavynode'] %}
-INDXS=$(curl -s -XGET -k -L https://{{ NODEIP }}:9200/_cat/indices?v | egrep 'logstash|elastalert|so-' | awk '{ print $3 }')
-{% else %}
-INDXS=$(curl -s -XGET -L {{ NODEIP }}:9200/_cat/indices?v | egrep 'logstash|elastalert|so-' | awk '{ print $3 }')
-{% endif %}
-for INDX in ${INDXS}
-do
-    {% if grains['role'] in ['so-node','so-heavynode'] %}
-    curl -XDELETE -k -L https://"{{ NODEIP }}:9200/${INDX}" > /dev/null 2>&1
-    {% else %}
-    curl -XDELETE -L "{{ NODEIP }}:9200/${INDX}" > /dev/null 2>&1
-    {% endif %}
-done
-
-#Start Logstash/Filebeat
-if [ ! -z "$FB_ENABLED" ]; then
-
-        /usr/sbin/so-filebeat-start
-
+# Delete log data
+if [ ! -z "$DELETE_LOG_DATA" ]; then
+	echo "Deleting log data ..."
+	DATASTREAMS=$(/usr/sbin/so-elasticsearch-query _data_stream | jq -r '.[] |.[].name')
+	for DATASTREAM in ${DATASTREAMS}
+	do
+		# Delete the data stream
+        	echo "Deleting $DATASTREAM..."
+        	/usr/sbin/so-elasticsearch-query _data_stream/${DATASTREAM} -XDELETE > /dev/null 2>&1
+	done
 fi
 
-if [ ! -z "$LS_ENABLED" ]; then
+if [ -z "$DONT_STOP_SERVICES" ]; then
+	#Start Logstash
+	if [ ! -z "$LS_ENABLED" ]; then
+        	/usr/sbin/so-logstash-start
 
-        /usr/sbin/so-logstash-start
+	fi
 
+	#Start Elastic Fleet
+	#if [ ! -z "$EF_ENABLED" ]; then
+        # 	/usr/sbin/so-elastic-fleet-start
+	#fi
+
+	#Start Elastalert
+	if [ ! -z "$EA_ENABLED" ]; then
+        	/usr/sbin/so-elastalert-start
+	fi
+
+	# Start Elastic Agent
+	/usr/bin/elastic-agent restart
 fi
-
-if [ ! -z "$EA_ENABLED" ]; then
-
-        /usr/sbin/so-elastalert-start
-
-fi
-

salt/common/tools/sbin/so-elastic-diagnose

@@ -1,19 +1,11 @@
 #!/bin/bash
 
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+
 
 # Source common settings
 . /usr/sbin/so-common

salt/common/tools/sbin/so-elastic-fleet-agent-policy-delete

@@ -0,0 +1,19 @@
+#!/bin/bash
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+. /usr/sbin/so-common
+
+POLICY_ID=$1
+
+# Let's snag a cookie from Kibana
+SESSIONCOOKIE=$(curl -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}')
+
+echo "Deleting agent policy $POLICY_ID..."
+
+# Delete agent policy
+curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/agent_policies/delete" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d"{\"agentPolicyId\": \"$POLICY_ID\"}"
+
+echo

salt/common/tools/sbin/so-elastic-fleet-agent-policy-list

@@ -0,0 +1,17 @@
+#!/bin/bash
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+. /usr/sbin/so-common
+
+# Let's snag a cookie from Kibana
+SESSIONCOOKIE=$(curl -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}')
+
+echo "Setting up default Security Onion package policies for Elastic Agent..."
+
+# List configured agent policies
+curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/agent_policies" | jq 
+
+echo

salt/common/tools/sbin/so-elastic-fleet-agent-policy-view

@@ -0,0 +1,19 @@
+#!/bin/bash
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+. /usr/sbin/so-common
+
+POLICY_ID=$1
+
+# Let's snag a cookie from Kibana
+SESSIONCOOKIE=$(curl -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}')
+
+echo "Viewing agent policy $POLICY_ID"
+
+# View agent policy
+curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/agent_policies/$POLICY_ID/full" | jq
+
+echo

salt/common/tools/sbin/so-elastic-fleet-data-streams-list

@@ -0,0 +1,17 @@
+#!/bin/bash
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+. /usr/sbin/so-common
+
+# Let's snag a cookie from Kibana
+SESSIONCOOKIE=$(curl -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}')
+
+echo "Retrieving data stream information..."
+
+# Retrieve data stream information
+curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/data_streams" | jq 
+
+echo

salt/common/tools/sbin/so-elastic-fleet-integration-policy-bulk-delete

@@ -0,0 +1,23 @@
+#!/bin/bash
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+. /usr/sbin/so-common
+
+POLICY_ID=$1
+
+# Let's snag a cookie from Kibana
+SESSIONCOOKIE=$(curl -q -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}')
+
+# Get integration policies relative to agent policy
+INTEGRATION_POLICY_IDS=$(curl -q -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/agent_policies/$POLICY_ID" | jq -r '.item.package_policies[].id')
+
+for i in $INTEGRATION_POLICY_IDS; do
+  # Delete integration policies
+  echo "Deleting integration policy: $i..."
+  curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies/delete" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d"{\"packagePolicyIds\": [\"$i\"], \"force\":true}";
+  echo
+  echo
+done

salt/common/tools/sbin/so-elastic-fleet-integration-policy-delete

@@ -0,0 +1,19 @@
+#!/bin/bash
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+. /usr/sbin/so-common
+
+POLICY_ID=$1
+
+# Let's snag a cookie from Kibana
+SESSIONCOOKIE=$(curl -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}')
+
+echo "Deleting integration policy $POLICY_ID..."
+
+# List configured package policies
+curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies/delete" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d"{\"packagePolicyIds\": [\"$POLICY_ID\"]}"
+
+echo

salt/common/tools/sbin/so-elastic-fleet-integration-policy-list

@@ -0,0 +1,17 @@
+#!/bin/bash
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+. /usr/sbin/so-common
+
+# Let's snag a cookie from Kibana
+SESSIONCOOKIE=$(curl -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}')
+
+echo "Setting up default Security Onion package policies for Elastic Agent..."
+
+# List configured package policies
+curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/package_policies" | jq 
+
+echo

salt/common/tools/sbin/so-elastic-fleet-integration-policy-load

@@ -0,0 +1,21 @@
+#!/bin/bash
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+. /usr/sbin/so-common
+
+# Initial Endpoints
+for INTEGRATION in /opt/so/saltstack/default/salt/elasticfleet/files/integrations/endpoints-initial/*.json
+do
+  printf "\n\nInitial Endpoint Policy - Loading $INTEGRATION\n"
+  elastic_fleet_integration_create "@$INTEGRATION"
+done
+
+# Grid Nodes
+for INTEGRATION in /opt/so/saltstack/default/salt/elasticfleet/files/integrations/grid-nodes/*.json
+do
+  printf "\n\nGrid Nodes Policy - Loading $INTEGRATION\n"
+  elastic_fleet_integration_create "@$INTEGRATION"
+done