CSEC_PUBLIC/securityonion
1
0
Fork 0
mirror of https://github.com/Security-Onion-Solutions/securityonion.git synced 2026-04-01 02:13:57 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #15696 from Security-Onion-Solutions/3/dev

Browse Source 
3.0.0
This commit is contained in:
Mike Reeves
2026-03-31 13:47:03 -04:00
committed by GitHub
parent cd0d88e2c0 cd886dd0f9
commit dcc67b9b8f
466 changed files with 81283 additions and 50457 deletions
Download Patch File Download Diff File Expand all files Collapse all files

546
.github/.gitleaks.toml vendored
View File

@@ -1,546 +0,0 @@
title = "gitleaks config"
# Gitleaks rules are defined by regular expressions and entropy ranges.
# Some secrets have unique signatures which make detecting those secrets easy.
# Examples of those secrets would be GitLab Personal Access Tokens, AWS keys, and GitHub Access Tokens.
# All these examples have defined prefixes like `glpat`, `AKIA`, `ghp_`, etc.
#
# Other secrets might just be a hash which means we need to write more complex rules to verify
# that what we are matching is a secret.
#
# Here is an example of a semi-generic secret
#
# discord_client_secret = "8dyfuiRyq=vVc3RRr_edRk-fK__JItpZ"
#
# We can write a regular expression to capture the variable name (identifier),
# the assignment symbol (like '=' or ':='), and finally the actual secret.
# The structure of a rule to match this example secret is below:
#
# Beginning string
# quotation
# │ End string quotation
# │ │
# ▼ ▼
# (?i)(discord[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9=_\-]{32})['\"]
#
# ▲ ▲ ▲
# │ │ │
# │ │ │
# identifier assignment symbol
# Secret
#
[[rules]]
id = "gitlab-pat"
description = "GitLab Personal Access Token"
regex = '''glpat-[0-9a-zA-Z\-\_]{20}'''
[[rules]]
id = "aws-access-token"
description = "AWS"
regex = '''(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}'''
# Cryptographic keys
[[rules]]
id = "PKCS8-PK"
description = "PKCS8 private key"
regex = '''-----BEGIN PRIVATE KEY-----'''
[[rules]]
id = "RSA-PK"
description = "RSA private key"
regex = '''-----BEGIN RSA PRIVATE KEY-----'''
[[rules]]
id = "OPENSSH-PK"
description = "SSH private key"
regex = '''-----BEGIN OPENSSH PRIVATE KEY-----'''
[[rules]]
id = "PGP-PK"
description = "PGP private key"
regex = '''-----BEGIN PGP PRIVATE KEY BLOCK-----'''
[[rules]]
id = "github-pat"
description = "GitHub Personal Access Token"
regex = '''ghp_[0-9a-zA-Z]{36}'''
[[rules]]
id = "github-oauth"
description = "GitHub OAuth Access Token"
regex = '''gho_[0-9a-zA-Z]{36}'''
[[rules]]
id = "SSH-DSA-PK"
description = "SSH (DSA) private key"
regex = '''-----BEGIN DSA PRIVATE KEY-----'''
[[rules]]
id = "SSH-EC-PK"
description = "SSH (EC) private key"
regex = '''-----BEGIN EC PRIVATE KEY-----'''
[[rules]]
id = "github-app-token"
description = "GitHub App Token"
regex = '''(ghu|ghs)_[0-9a-zA-Z]{36}'''
[[rules]]
id = "github-refresh-token"
description = "GitHub Refresh Token"
regex = '''ghr_[0-9a-zA-Z]{76}'''
[[rules]]
id = "shopify-shared-secret"
description = "Shopify shared secret"
regex = '''shpss_[a-fA-F0-9]{32}'''
[[rules]]
id = "shopify-access-token"
description = "Shopify access token"
regex = '''shpat_[a-fA-F0-9]{32}'''
[[rules]]
id = "shopify-custom-access-token"
description = "Shopify custom app access token"
regex = '''shpca_[a-fA-F0-9]{32}'''
[[rules]]
id = "shopify-private-app-access-token"
description = "Shopify private app access token"
regex = '''shppa_[a-fA-F0-9]{32}'''
[[rules]]
id = "slack-access-token"
description = "Slack token"
regex = '''xox[baprs]-([0-9a-zA-Z]{10,48})?'''
[[rules]]
id = "stripe-access-token"
description = "Stripe"
regex = '''(?i)(sk|pk)_(test|live)_[0-9a-z]{10,32}'''
[[rules]]
id = "pypi-upload-token"
description = "PyPI upload token"
regex = '''pypi-AgEIcHlwaS5vcmc[A-Za-z0-9\-_]{50,1000}'''
[[rules]]
id = "gcp-service-account"
description = "Google (GCP) Service-account"
regex = '''\"type\": \"service_account\"'''
[[rules]]
id = "heroku-api-key"
description = "Heroku API Key"
regex = ''' (?i)(heroku[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([0-9A-F]{8}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{12})['\"]'''
secretGroup = 3
[[rules]]
id = "slack-web-hook"
description = "Slack Webhook"
regex = '''https://hooks.slack.com/services/T[a-zA-Z0-9_]{8}/B[a-zA-Z0-9_]{8,12}/[a-zA-Z0-9_]{24}'''
[[rules]]
id = "twilio-api-key"
description = "Twilio API Key"
regex = '''SK[0-9a-fA-F]{32}'''
[[rules]]
id = "age-secret-key"
description = "Age secret key"
regex = '''AGE-SECRET-KEY-1[QPZRY9X8GF2TVDW0S3JN54KHCE6MUA7L]{58}'''
[[rules]]
id = "facebook-token"
description = "Facebook token"
regex = '''(?i)(facebook[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-f0-9]{32})['\"]'''
secretGroup = 3
[[rules]]
id = "twitter-token"
description = "Twitter token"
regex = '''(?i)(twitter[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-f0-9]{35,44})['\"]'''
secretGroup = 3
[[rules]]
id = "adobe-client-id"
description = "Adobe Client ID (Oauth Web)"
regex = '''(?i)(adobe[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-f0-9]{32})['\"]'''
secretGroup = 3
[[rules]]
id = "adobe-client-secret"
description = "Adobe Client Secret"
regex = '''(p8e-)(?i)[a-z0-9]{32}'''
[[rules]]
id = "alibaba-access-key-id"
description = "Alibaba AccessKey ID"
regex = '''(LTAI)(?i)[a-z0-9]{20}'''
[[rules]]
id = "alibaba-secret-key"
description = "Alibaba Secret Key"
regex = '''(?i)(alibaba[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{30})['\"]'''
secretGroup = 3
[[rules]]
id = "asana-client-id"
description = "Asana Client ID"
regex = '''(?i)(asana[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([0-9]{16})['\"]'''
secretGroup = 3
[[rules]]
id = "asana-client-secret"
description = "Asana Client Secret"
regex = '''(?i)(asana[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{32})['\"]'''
secretGroup = 3
[[rules]]
id = "atlassian-api-token"
description = "Atlassian API token"
regex = '''(?i)(atlassian[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{24})['\"]'''
secretGroup = 3
[[rules]]
id = "bitbucket-client-id"
description = "Bitbucket client ID"
regex = '''(?i)(bitbucket[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{32})['\"]'''
secretGroup = 3
[[rules]]
id = "bitbucket-client-secret"
description = "Bitbucket client secret"
regex = '''(?i)(bitbucket[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9_\-]{64})['\"]'''
secretGroup = 3
[[rules]]
id = "beamer-api-token"
description = "Beamer API token"
regex = '''(?i)(beamer[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](b_[a-z0-9=_\-]{44})['\"]'''
secretGroup = 3
[[rules]]
id = "clojars-api-token"
description = "Clojars API token"
regex = '''(CLOJARS_)(?i)[a-z0-9]{60}'''
[[rules]]
id = "contentful-delivery-api-token"
description = "Contentful delivery API token"
regex = '''(?i)(contentful[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9\-=_]{43})['\"]'''
secretGroup = 3
[[rules]]
id = "databricks-api-token"
description = "Databricks API token"
regex = '''dapi[a-h0-9]{32}'''
[[rules]]
id = "discord-api-token"
description = "Discord API key"
regex = '''(?i)(discord[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-h0-9]{64})['\"]'''
secretGroup = 3
[[rules]]
id = "discord-client-id"
description = "Discord client ID"
regex = '''(?i)(discord[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([0-9]{18})['\"]'''
secretGroup = 3
[[rules]]
id = "discord-client-secret"
description = "Discord client secret"
regex = '''(?i)(discord[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9=_\-]{32})['\"]'''
secretGroup = 3
[[rules]]
id = "doppler-api-token"
description = "Doppler API token"
regex = '''['\"](dp\.pt\.)(?i)[a-z0-9]{43}['\"]'''
[[rules]]
id = "dropbox-api-secret"
description = "Dropbox API secret/key"
regex = '''(?i)(dropbox[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{15})['\"]'''
[[rules]]
id = "dropbox--api-key"
description = "Dropbox API secret/key"
regex = '''(?i)(dropbox[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{15})['\"]'''
[[rules]]
id = "dropbox-short-lived-api-token"
description = "Dropbox short lived API token"
regex = '''(?i)(dropbox[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](sl\.[a-z0-9\-=_]{135})['\"]'''
[[rules]]
id = "dropbox-long-lived-api-token"
description = "Dropbox long lived API token"
regex = '''(?i)(dropbox[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"][a-z0-9]{11}(AAAAAAAAAA)[a-z0-9\-_=]{43}['\"]'''
[[rules]]
id = "duffel-api-token"
description = "Duffel API token"
regex = '''['\"]duffel_(test|live)_(?i)[a-z0-9_-]{43}['\"]'''
[[rules]]
id = "dynatrace-api-token"
description = "Dynatrace API token"
regex = '''['\"]dt0c01\.(?i)[a-z0-9]{24}\.[a-z0-9]{64}['\"]'''
[[rules]]
id = "easypost-api-token"
description = "EasyPost API token"
regex = '''['\"]EZAK(?i)[a-z0-9]{54}['\"]'''
[[rules]]
id = "easypost-test-api-token"
description = "EasyPost test API token"
regex = '''['\"]EZTK(?i)[a-z0-9]{54}['\"]'''
[[rules]]
id = "fastly-api-token"
description = "Fastly API token"
regex = '''(?i)(fastly[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9\-=_]{32})['\"]'''
secretGroup = 3
[[rules]]
id = "finicity-client-secret"
description = "Finicity client secret"
regex = '''(?i)(finicity[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{20})['\"]'''
secretGroup = 3
[[rules]]
id = "finicity-api-token"
description = "Finicity API token"
regex = '''(?i)(finicity[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-f0-9]{32})['\"]'''
secretGroup = 3
[[rules]]
id = "flutterwave-public-key"
description = "Flutterwave public key"
regex = '''FLWPUBK_TEST-(?i)[a-h0-9]{32}-X'''
[[rules]]
id = "flutterwave-secret-key"
description = "Flutterwave secret key"
regex = '''FLWSECK_TEST-(?i)[a-h0-9]{32}-X'''
[[rules]]
id = "flutterwave-enc-key"
description = "Flutterwave encrypted key"
regex = '''FLWSECK_TEST[a-h0-9]{12}'''
[[rules]]
id = "frameio-api-token"
description = "Frame.io API token"
regex = '''fio-u-(?i)[a-z0-9\-_=]{64}'''
[[rules]]
id = "gocardless-api-token"
description = "GoCardless API token"
regex = '''['\"]live_(?i)[a-z0-9\-_=]{40}['\"]'''
[[rules]]
id = "grafana-api-token"
description = "Grafana API token"
regex = '''['\"]eyJrIjoi(?i)[a-z0-9\-_=]{72,92}['\"]'''
[[rules]]
id = "hashicorp-tf-api-token"
description = "HashiCorp Terraform user/org API token"
regex = '''['\"](?i)[a-z0-9]{14}\.atlasv1\.[a-z0-9\-_=]{60,70}['\"]'''
[[rules]]
id = "hubspot-api-token"
description = "HubSpot API token"
regex = '''(?i)(hubspot[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-h0-9]{8}-[a-h0-9]{4}-[a-h0-9]{4}-[a-h0-9]{4}-[a-h0-9]{12})['\"]'''
secretGroup = 3
[[rules]]
id = "intercom-api-token"
description = "Intercom API token"
regex = '''(?i)(intercom[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9=_]{60})['\"]'''
secretGroup = 3
[[rules]]
id = "intercom-client-secret"
description = "Intercom client secret/ID"
regex = '''(?i)(intercom[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-h0-9]{8}-[a-h0-9]{4}-[a-h0-9]{4}-[a-h0-9]{4}-[a-h0-9]{12})['\"]'''
secretGroup = 3
[[rules]]
id = "ionic-api-token"
description = "Ionic API token"
regex = '''(?i)(ionic[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](ion_[a-z0-9]{42})['\"]'''
[[rules]]
id = "linear-api-token"
description = "Linear API token"
regex = '''lin_api_(?i)[a-z0-9]{40}'''
[[rules]]
id = "linear-client-secret"
description = "Linear client secret/ID"
regex = '''(?i)(linear[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-f0-9]{32})['\"]'''
secretGroup = 3
[[rules]]
id = "lob-api-key"
description = "Lob API Key"
regex = '''(?i)(lob[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]((live|test)_[a-f0-9]{35})['\"]'''
secretGroup = 3
[[rules]]
id = "lob-pub-api-key"
description = "Lob Publishable API Key"
regex = '''(?i)(lob[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]((test|live)_pub_[a-f0-9]{31})['\"]'''
secretGroup = 3
[[rules]]
id = "mailchimp-api-key"
description = "Mailchimp API key"
regex = '''(?i)(mailchimp[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-f0-9]{32}-us20)['\"]'''
secretGroup = 3
[[rules]]
id = "mailgun-private-api-token"
description = "Mailgun private API token"
regex = '''(?i)(mailgun[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](key-[a-f0-9]{32})['\"]'''
secretGroup = 3
[[rules]]
id = "mailgun-pub-key"
description = "Mailgun public validation key"
regex = '''(?i)(mailgun[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](pubkey-[a-f0-9]{32})['\"]'''
secretGroup = 3
[[rules]]
id = "mailgun-signing-key"
description = "Mailgun webhook signing key"
regex = '''(?i)(mailgun[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-h0-9]{32}-[a-h0-9]{8}-[a-h0-9]{8})['\"]'''
secretGroup = 3
[[rules]]
id = "mapbox-api-token"
description = "Mapbox API token"
regex = '''(?i)(pk\.[a-z0-9]{60}\.[a-z0-9]{22})'''
[[rules]]
id = "messagebird-api-token"
description = "MessageBird API token"
regex = '''(?i)(messagebird[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{25})['\"]'''
secretGroup = 3
[[rules]]
id = "messagebird-client-id"
description = "MessageBird API client ID"
regex = '''(?i)(messagebird[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-h0-9]{8}-[a-h0-9]{4}-[a-h0-9]{4}-[a-h0-9]{4}-[a-h0-9]{12})['\"]'''
secretGroup = 3
[[rules]]
id = "new-relic-user-api-key"
description = "New Relic user API Key"
regex = '''['\"](NRAK-[A-Z0-9]{27})['\"]'''
[[rules]]
id = "new-relic-user-api-id"
description = "New Relic user API ID"
regex = '''(?i)(newrelic[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([A-Z0-9]{64})['\"]'''
secretGroup = 3
[[rules]]
id = "new-relic-browser-api-token"
description = "New Relic ingest browser API token"
regex = '''['\"](NRJS-[a-f0-9]{19})['\"]'''
[[rules]]
id = "npm-access-token"
description = "npm access token"
regex = '''['\"](npm_(?i)[a-z0-9]{36})['\"]'''
[[rules]]
id = "planetscale-password"
description = "PlanetScale password"
regex = '''pscale_pw_(?i)[a-z0-9\-_\.]{43}'''
[[rules]]
id = "planetscale-api-token"
description = "PlanetScale API token"
regex = '''pscale_tkn_(?i)[a-z0-9\-_\.]{43}'''
[[rules]]
id = "postman-api-token"
description = "Postman API token"
regex = '''PMAK-(?i)[a-f0-9]{24}\-[a-f0-9]{34}'''
[[rules]]
id = "pulumi-api-token"
description = "Pulumi API token"
regex = '''pul-[a-f0-9]{40}'''
[[rules]]
id = "rubygems-api-token"
description = "Rubygem API token"
regex = '''rubygems_[a-f0-9]{48}'''
[[rules]]
id = "sendgrid-api-token"
description = "SendGrid API token"
regex = '''SG\.(?i)[a-z0-9_\-\.]{66}'''
[[rules]]
id = "sendinblue-api-token"
description = "Sendinblue API token"
regex = '''xkeysib-[a-f0-9]{64}\-(?i)[a-z0-9]{16}'''
[[rules]]
id = "shippo-api-token"
description = "Shippo API token"
regex = '''shippo_(live|test)_[a-f0-9]{40}'''
[[rules]]
id = "linkedin-client-secret"
description = "LinkedIn Client secret"
regex = '''(?i)(linkedin[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z]{16})['\"]'''
secretGroup = 3
[[rules]]
id = "linkedin-client-id"
description = "LinkedIn Client ID"
regex = '''(?i)(linkedin[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{14})['\"]'''
secretGroup = 3
[[rules]]
id = "twitch-api-token"
description = "Twitch API token"
regex = '''(?i)(twitch[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{30})['\"]'''
secretGroup = 3
[[rules]]
id = "typeform-api-token"
description = "Typeform API token"
regex = '''(?i)(typeform[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}(tfp_[a-z0-9\-_\.=]{59})'''
secretGroup = 3
[[rules]]
id = "generic-api-key"
description = "Generic API Key"
regex = '''(?i)((key|api[^Version]|token|secret|password)[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([0-9a-zA-Z\-_=]{8,64})['\"]'''
entropy = 3.7
secretGroup = 4
[allowlist]
description = "global allow lists"
regexes = ['''219-09-9999''', '''078-05-1120''', '''(9[0-9]{2}|666)-\d{2}-\d{4}''', '''RPM-GPG-KEY.*''', '''.*:.*StrelkaHexDump.*''', '''.*:.*PLACEHOLDER.*''', '''ssl_.*password''', '''integration_key\s=\s"so-logs-"''']
paths = [
'''gitleaks.toml''',
'''(.*?)(jpg|gif|doc|pdf|bin|svg|socket)$''',
'''(go.mod|go.sum)$''',
'''salt/nginx/files/enterprise-attack.json''',
'''(.*?)whl$'''
]

9
.github/DISCUSSION_TEMPLATE/2-4.yml vendored
View File

@@ -2,13 +2,11 @@ body:
- type: markdown
attributes:
value: |
⚠️ This category is solely for conversations related to Security Onion 2.4 ⚠️
If your organization needs more immediate, enterprise grade professional support, with one-on-one virtual meetings and screensharing, contact us via our website: https://securityonion.com/support
- type: dropdown
attributes:
label: Version
description: Which version of Security Onion 2.4.x are you asking about?
description: Which version of Security Onion are you asking about?
options:
-
- 2.4.10
@@ -33,6 +31,9 @@ body:
- 2.4.180
- 2.4.190
- 2.4.200
- 2.4.201
- 2.4.210
- 2.4.211
- Other (please provide detail below)
validations:
required: true
@@ -94,7 +95,7 @@ body:
attributes:
label: Hardware Specs
description: >
Does your hardware meet or exceed the minimum requirements for your installation type as shown at https://docs.securityonion.net/en/2.4/hardware.html?
Does your hardware meet or exceed the minimum requirements for your installation type as shown at https://securityonion.net/docs/hardware?
options:
-
- Meets minimum requirements

177
.github/DISCUSSION_TEMPLATE/3-0.yml vendored Normal file
View File

@@ -0,0 +1,177 @@
body:
- type: markdown
attributes:
value: |
If your organization needs more immediate, enterprise grade professional support, with one-on-one virtual meetings and screensharing, contact us via our website: https://securityonion.com/support
- type: dropdown
attributes:
label: Version
description: Which version of Security Onion are you asking about?
options:
-
- 3.0.0
- Other (please provide detail below)
validations:
required: true
- type: dropdown
attributes:
label: Installation Method
description: How did you install Security Onion?
options:
-
- Security Onion ISO image
- Cloud image (Amazon, Azure, Google)
- Network installation on Oracle 9 (unsupported)
- Other (please provide detail below)
validations:
required: true
- type: dropdown
attributes:
label: Description
description: >
Is this discussion about installation, configuration, upgrading, or other?
options:
-
- installation
- configuration
- upgrading
- other (please provide detail below)
validations:
required: true
- type: dropdown
attributes:
label: Installation Type
description: >
When you installed, did you choose Import, Eval, Standalone, Distributed, or something else?
options:
-
- Import
- Eval
- Standalone
- Distributed
- other (please provide detail below)
validations:
required: true
- type: dropdown
attributes:
label: Location
description: >
Is this deployment in the cloud, on-prem with Internet access, or airgap?
options:
-
- cloud
- on-prem with Internet access
- airgap
- other (please provide detail below)
validations:
required: true
- type: dropdown
attributes:
label: Hardware Specs
description: >
Does your hardware meet or exceed the minimum requirements for your installation type as shown at https://securityonion.net/docs/hardware?
options:
-
- Meets minimum requirements
- Exceeds minimum requirements
- Does not meet minimum requirements
- other (please provide detail below)
validations:
required: true
- type: input
attributes:
label: CPU
description: How many CPU cores do you have?
validations:
required: true
- type: input
attributes:
label: RAM
description: How much RAM do you have?
validations:
required: true
- type: input
attributes:
label: Storage for /
description: How much storage do you have for the / partition?
validations:
required: true
- type: input
attributes:
label: Storage for /nsm
description: How much storage do you have for the /nsm partition?
validations:
required: true
- type: dropdown
attributes:
label: Network Traffic Collection
description: >
Are you collecting network traffic from a tap or span port?
options:
-
- tap
- span port
- other (please provide detail below)
validations:
required: true
- type: dropdown
attributes:
label: Network Traffic Speeds
description: >
How much network traffic are you monitoring?
options:
-
- Less than 1Gbps
- 1Gbps to 10Gbps
- more than 10Gbps
validations:
required: true
- type: dropdown
attributes:
label: Status
description: >
Does SOC Grid show all services on all nodes as running OK?
options:
-
- Yes, all services on all nodes are running OK
- No, one or more services are failed (please provide detail below)
validations:
required: true
- type: dropdown
attributes:
label: Salt Status
description: >
Do you get any failures when you run "sudo salt-call state.highstate"?
options:
-
- Yes, there are salt failures (please provide detail below)
- No, there are no failures
validations:
required: true
- type: dropdown
attributes:
label: Logs
description: >
Are there any additional clues in /opt/so/log/?
options:
-
- Yes, there are additional clues in /opt/so/log/ (please provide detail below)
- No, there are no additional clues
validations:
required: true
- type: textarea
attributes:
label: Detail
description: Please read our discussion guidelines at https://github.com/Security-Onion-Solutions/securityonion/discussions/1720 and then provide detailed information to help us help you.
placeholder: |-
STOP! Before typing, please read our discussion guidelines at https://github.com/Security-Onion-Solutions/securityonion/discussions/1720 in their entirety!
If your organization needs more immediate, enterprise grade professional support, with one-on-one virtual meetings and screensharing, contact us via our website: https://securityonion.com/support
validations:
required: true
- type: checkboxes
attributes:
label: Guidelines
options:
- label: I have read the discussion guidelines at https://github.com/Security-Onion-Solutions/securityonion/discussions/1720 and assert that I have followed the guidelines.
required: true

17
.github/workflows/leaktest.yml vendored
View File

@@ -1,17 +0,0 @@
name: leak-test
on: [pull_request]
jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
with:
fetch-depth: '0'
- name: Gitleaks
uses: gitleaks/gitleaks-action@v1.6.0
with:
config-path: .github/.gitleaks.toml

2
.github/workflows/pythontest.yml vendored
View File

@@ -13,7 +13,7 @@ jobs:
strategy:
fail-fast: false
matrix:
python-version: ["3.13"]
python-version: ["3.14"]
python-code-path: ["salt/sensoroni/files/analyzers", "salt/manager/tools/sbin"]
steps:

28
DOWNLOAD_AND_VERIFY_ISO.md
View File

@@ -1,46 +1,46 @@
### 2.4.201-20260114 ISO image released on 2026/1/15
### 3.0.0-20260331 ISO image released on 2026/03/31
### Download and Verify
2.4.201-20260114 ISO image:
https://download.securityonion.net/file/securityonion/securityonion-2.4.201-20260114.iso
3.0.0-20260331 ISO image:
https://download.securityonion.net/file/securityonion/securityonion-3.0.0-20260331.iso
MD5: 20E926E433203798512EF46E590C89B9
SHA1: 779E4084A3E1A209B494493B8F5658508B6014FA
SHA256: 3D10E7C885AEC5C5D4F4E50F9644FF9728E8C0A2E36EBB8C96B32569685A7C40
MD5: ECD318A1662A6FDE0EF213F5A9BD4B07
SHA1: E55BE314440CCF3392DC0B06BC5E270B43176D9C
SHA256: 7FC47405E335CBE5C2B6C51FE7AC60248F35CBE504907B8B5A33822B23F8F4D5
Signature for ISO image:
https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.201-20260114.iso.sig
https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.0.0-20260331.iso.sig
Signing key:
https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.4/main/KEYS
https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/3/main/KEYS
For example, here are the steps you can use on most Linux distributions to download and verify our Security Onion ISO image.
Download and import the signing key:
```
wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.4/main/KEYS -O - | gpg --import -
wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/3/main/KEYS -O - | gpg --import -
```
Download the signature file for the ISO:
```
wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.201-20260114.iso.sig
wget https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.0.0-20260331.iso.sig
```
Download the ISO image:
```
wget https://download.securityonion.net/file/securityonion/securityonion-2.4.201-20260114.iso
wget https://download.securityonion.net/file/securityonion/securityonion-3.0.0-20260331.iso
```
Verify the downloaded ISO image using the signature file:
```
gpg --verify securityonion-2.4.201-20260114.iso.sig securityonion-2.4.201-20260114.iso
gpg --verify securityonion-3.0.0-20260331.iso.sig securityonion-3.0.0-20260331.iso
```
The output should show "Good signature" and the Primary key fingerprint should match what's shown below:
```
gpg: Signature made Wed 14 Jan 2026 05:23:39 PM EST using RSA key ID FE507013
gpg: Signature made Mon 30 Mar 2026 06:22:14 PM EDT using RSA key ID FE507013
gpg: Good signature from "Security Onion Solutions, LLC <info@securityonionsolutions.com>"
gpg: WARNING: This key is not certified with a trusted signature!
gpg: There is no indication that the signature belongs to the owner.
@@ -50,4 +50,4 @@ Primary key fingerprint: C804 A93D 36BE 0C73 3EA1 9644 7C10 60B7 FE50 7013
If it fails to verify, try downloading again. If it still fails to verify, try downloading from another computer or another network.
Once you've verified the ISO image, you're ready to proceed to our Installation guide:
https://docs.securityonion.net/en/2.4/installation.html
https://securityonion.net/docs/installation

66
README.md
View File

@@ -1,50 +1,58 @@
## Security Onion 2.4
<p align="center">
<img src="https://securityonionsolutions.com/logo/logo-so-onion-dark.svg" width="400" alt="Security Onion Logo">
</p>
Security Onion 2.4 is here!
# Security Onion
## Screenshots
Security Onion is a free and open Linux distribution for threat hunting, enterprise security monitoring, and log management. It includes a comprehensive suite of tools designed to work together to provide visibility into your network and host activity.
Alerts
![Alerts](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/50_alerts.png)
## ✨ Features
Dashboards
![Dashboards](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/53_dashboards.png)
Security Onion includes everything you need to monitor your network and host systems:
Hunt
![Hunt](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/56_hunt.png)
* **Security Onion Console (SOC)**: A unified web interface for analyzing security events and managing your grid.
* **Elastic Stack**: Powerful search backed by Elasticsearch.
* **Intrusion Detection**: Network-based IDS with Suricata and host-based monitoring with Elastic Fleet.
* **Network Metadata**: Detailed network metadata generated by Zeek or Suricata.
* **Full Packet Capture**: Retain and analyze raw network traffic with Suricata PCAP.
Detections
![Detections](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/57_detections.png)
## ⭐ Security Onion Pro
PCAP
![PCAP](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/62_pcap.png)
For organizations and enterprises requiring advanced capabilities, **Security Onion Pro** offers additional features designed for scale and efficiency:
Grid
![Grid](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/75_grid.png)
* **Onion AI**: Leverage powerful AI-driven insights to accelerate your analysis and investigations.
* **Enterprise Features**: Enhanced tools and integrations tailored for enterprise-grade security operations.
Config
![Config](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/87_config.png)
For more information, visit the [Security Onion Pro](https://securityonionsolutions.com/pro) page.
### Release Notes
## ☁️ Cloud Deployment
https://docs.securityonion.net/en/2.4/release-notes.html
Security Onion is available and ready to deploy in the **AWS**, **Azure**, and **Google Cloud (GCP)** marketplaces.
### Requirements
## 🚀 Getting Started
https://docs.securityonion.net/en/2.4/hardware.html
| Goal | Resource |
| :--- | :--- |
| **Download** | [Security Onion ISO](https://securityonion.net/docs/download) |
| **Requirements** | [Hardware Guide](https://securityonion.net/docs/hardware) |
| **Install** | [Installation Instructions](https://securityonion.net/docs/installation) |
| **What's New** | [Release Notes](https://securityonion.net/docs/release-notes) |
### Download
## 📖 Documentation & Support
https://docs.securityonion.net/en/2.4/download.html
For more detailed information, please visit our [Documentation](https://docs.securityonion.net).
### Installation
* **FAQ**: [Frequently Asked Questions](https://securityonion.net/docs/faq)
* **Community**: [Discussions & Support](https://securityonion.net/docs/community-support)
* **Training**: [Official Training](https://securityonion.net/training)
https://docs.securityonion.net/en/2.4/installation.html
## 🤝 Contributing
### FAQ
We welcome contributions! Please see our [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines on how to get involved.
https://docs.securityonion.net/en/2.4/faq.html
## 🛡️ License
### Feedback
Security Onion is licensed under the terms of the license found in the [LICENSE](LICENSE) file.
https://docs.securityonion.net/en/2.4/community-support.html
---
*Built with 🧅 by Security Onion Solutions.*

1
SECURITY.md
View File

@@ -4,6 +4,7 @@
| Version | Supported |
| ------- | ------------------ |
| 3.x | :white_check_mark: |
| 2.4.x | :white_check_mark: |
| 2.3.x | :x: |
| 16.04.x | :x: |

2
VERSION
View File

@@ -1 +1 @@
UNRELEASED
3.0.0

2
pillar/ca/init.sls Normal file
View File

@@ -0,0 +1,2 @@
ca:
server:

11
pillar/top.sls
View File

@@ -1,5 +1,6 @@
base:
'*':
- ca
- global.soc_global
- global.adv_global
- docker.soc_docker
@@ -86,8 +87,6 @@ base:
- zeek.adv_zeek
- bpf.soc_bpf
- bpf.adv_bpf
- pcap.soc_pcap
- pcap.adv_pcap
- suricata.soc_suricata
- suricata.adv_suricata
- minions.{{ grains.id }}
@@ -133,8 +132,6 @@ base:
- zeek.adv_zeek
- bpf.soc_bpf
- bpf.adv_bpf
- pcap.soc_pcap
- pcap.adv_pcap
- suricata.soc_suricata
- suricata.adv_suricata
- minions.{{ grains.id }}
@@ -184,8 +181,6 @@ base:
- zeek.adv_zeek
- bpf.soc_bpf
- bpf.adv_bpf
- pcap.soc_pcap
- pcap.adv_pcap
- suricata.soc_suricata
- suricata.adv_suricata
- minions.{{ grains.id }}
@@ -208,8 +203,6 @@ base:
- zeek.adv_zeek
- bpf.soc_bpf
- bpf.adv_bpf
- pcap.soc_pcap
- pcap.adv_pcap
- suricata.soc_suricata
- suricata.adv_suricata
- strelka.soc_strelka
@@ -296,8 +289,6 @@ base:
- zeek.adv_zeek
- bpf.soc_bpf
- bpf.adv_bpf
- pcap.soc_pcap
- pcap.adv_pcap
- suricata.soc_suricata
- suricata.adv_suricata
- strelka.soc_strelka

20
salt/_modules/needs_restarting.py
View File

@@ -1,24 +1,14 @@
from os import path
import subprocess
def check():
osfam = __grains__['os_family']
retval = 'False'
if osfam == 'Debian':
if path.exists('/var/run/reboot-required'):
retval = 'True'
cmd = 'needs-restarting -r > /dev/null 2>&1'
elif osfam == 'RedHat':
cmd = 'needs-restarting -r > /dev/null 2>&1'
try:
needs_restarting = subprocess.check_call(cmd, shell=True)
except subprocess.CalledProcessError:
retval = 'True'
else:
retval = 'Unsupported OS: %s' % os
try:
needs_restarting = subprocess.check_call(cmd, shell=True)
except subprocess.CalledProcessError:
retval = 'True'
return retval

28
salt/allowed_states.map.jinja
View File

@@ -15,11 +15,7 @@
'salt.minion-check',
'sensoroni',
'salt.lasthighstate',
'salt.minion'
] %}
{% set ssl_states = [
'ssl',
'salt.minion',
'telegraf',
'firewall',
'schedule',
@@ -28,7 +24,7 @@
{% set manager_states = [
'salt.master',
'ca',
'ca.server',
'registry',
'manager',
'nginx',
@@ -42,7 +38,6 @@
] %}
{% set sensor_states = [
'pcap',
'suricata',
'healthcheck',
'tcpreplay',
@@ -75,28 +70,24 @@
{# Map role-specific states #}
{% set role_states = {
'so-eval': (
ssl_states +
manager_states +
sensor_states +
elastic_stack_states | reject('equalto', 'logstash') | list
elastic_stack_states | reject('equalto', 'logstash') | list +
['logstash.ssl']
),
'so-heavynode': (
ssl_states +
sensor_states +
['elasticagent', 'elasticsearch', 'logstash', 'redis', 'nginx']
),
'so-idh': (
ssl_states +
['idh']
),
'so-import': (
ssl_states +
manager_states +
sensor_states | reject('equalto', 'strelka') | reject('equalto', 'healthcheck') | list +
['elasticsearch', 'elasticsearch.auth', 'kibana', 'kibana.secrets', 'strelka.manager']
['elasticsearch', 'elasticsearch.auth', 'kibana', 'kibana.secrets', 'logstash.ssl', 'strelka.manager']
),
'so-manager': (
ssl_states +
manager_states +
['salt.cloud', 'libvirt.packages', 'libvirt.ssh.users', 'strelka.manager'] +
stig_states +
@@ -104,7 +95,6 @@
elastic_stack_states
),
'so-managerhype': (
ssl_states +
manager_states +
['salt.cloud', 'strelka.manager', 'hypervisor', 'libvirt'] +
stig_states +
@@ -112,7 +102,6 @@
elastic_stack_states
),
'so-managersearch': (
ssl_states +
manager_states +
['salt.cloud', 'libvirt.packages', 'libvirt.ssh.users', 'strelka.manager'] +
stig_states +
@@ -120,12 +109,10 @@
elastic_stack_states
),
'so-searchnode': (
ssl_states +
['kafka.ca', 'kafka.ssl', 'elasticsearch', 'logstash', 'nginx'] +
stig_states
),
'so-standalone': (
ssl_states +
manager_states +
['salt.cloud', 'libvirt.packages', 'libvirt.ssh.users'] +
sensor_states +
@@ -134,29 +121,24 @@
elastic_stack_states
),
'so-sensor': (
ssl_states +
sensor_states +
['nginx'] +
stig_states
),
'so-fleet': (
ssl_states +
stig_states +
['logstash', 'nginx', 'healthcheck', 'elasticfleet']
),
'so-receiver': (
ssl_states +
kafka_states +
stig_states +
['logstash', 'redis']
),
'so-hypervisor': (
ssl_states +
stig_states +
['hypervisor', 'libvirt']
),
'so-desktop': (
['ssl', 'docker_clean', 'telegraf'] +
stig_states
)
} %}

4
salt/backup/soc_backup.yaml
View File

@@ -1,10 +1,10 @@
backup:
locations:
description: List of locations to back up to the destination.
helpLink: backup.html
helpLink: backup
global: True
destination:
description: Directory to store the configuration backups in.
helpLink: backup.html
helpLink: backup
global: True

6
salt/bpf/macros.jinja
View File

@@ -1,10 +1,12 @@
{% macro remove_comments(bpfmerged, app) %}
{# remove comments from the bpf #}
{% set app_list = [] %}
{% for bpf in bpfmerged[app] %}
{% if bpf.strip().startswith('#') %}
{% do bpfmerged[app].pop(loop.index0) %}
{% if not bpf.strip().startswith('#') %}
{% do app_list.append(bpf) %}
{% endif %}
{% endfor %}
{% do bpfmerged.update({app: app_list}) %}
{% endmacro %}

8
salt/bpf/pcap.map.jinja
View File

@@ -1,21 +1,15 @@
{% from 'vars/globals.map.jinja' import GLOBALS %}
{% set PCAP_BPF_STATUS = 0 %}
{% set STENO_BPF_COMPILED = "" %}
{% if GLOBALS.pcap_engine == "TRANSITION" %}
{% set PCAPBPF = ["ip and host 255.255.255.1 and port 1"] %}
{% else %}
{% import_yaml 'bpf/defaults.yaml' as BPFDEFAULTS %}
{% set BPFMERGED = salt['pillar.get']('bpf', BPFDEFAULTS.bpf, merge=True) %}
{% import 'bpf/macros.jinja' as MACROS %}
{{ MACROS.remove_comments(BPFMERGED, 'pcap') }}
{% set PCAPBPF = BPFMERGED.pcap %}
{% endif %}
{% if PCAPBPF %}
{% set PCAP_BPF_CALC = salt['cmd.run_all']('/usr/sbin/so-bpf-compile ' ~ GLOBALS.sensor.interface ~ ' ' ~ PCAPBPF|join(" "), cwd='/root') %}
{% set PCAP_BPF_CALC = salt['cmd.script']('salt://common/tools/sbin/so-bpf-compile', GLOBALS.sensor.interface + ' ' + PCAPBPF|join(" "),cwd='/root') %}
{% if PCAP_BPF_CALC['retcode'] == 0 %}
{% set PCAP_BPF_STATUS = 1 %}
{% set STENO_BPF_COMPILED = ",\\\"--filter=" + PCAP_BPF_CALC['stdout'] + "\\\"" %}
{% endif %}
{% endif %}

6
salt/bpf/soc_bpf.yaml
View File

@@ -3,14 +3,14 @@ bpf:
description: List of BPF filters to apply to the PCAP engine.
multiline: True
forcedType: "[]string"
helpLink: bpf.html
helpLink: bpf
suricata:
description: List of BPF filters to apply to Suricata. This will apply to alerts and, if enabled, to metadata and PCAP logs generated by Suricata.
multiline: True
forcedType: "[]string"
helpLink: bpf.html
helpLink: bpf
zeek:
description: List of BPF filters to apply to Zeek.
multiline: True
forcedType: "[]string"
helpLink: bpf.html
helpLink: bpf

2
salt/bpf/suricata.map.jinja
View File

@@ -9,7 +9,7 @@
{% set SURICATABPF = BPFMERGED.suricata %}
{% if SURICATABPF %}
{% set SURICATA_BPF_CALC = salt['cmd.run_all']('/usr/sbin/so-bpf-compile ' ~ GLOBALS.sensor.interface ~ ' ' ~ SURICATABPF|join(" "), cwd='/root') %}
{% set SURICATA_BPF_CALC = salt['cmd.script']('salt://common/tools/sbin/so-bpf-compile', GLOBALS.sensor.interface + ' ' + SURICATABPF|join(" "),cwd='/root') %}
{% if SURICATA_BPF_CALC['retcode'] == 0 %}
{% set SURICATA_BPF_STATUS = 1 %}
{% endif %}

2
salt/bpf/zeek.map.jinja
View File

@@ -9,7 +9,7 @@
{% set ZEEKBPF = BPFMERGED.zeek %}
{% if ZEEKBPF %}
{% set ZEEK_BPF_CALC = salt['cmd.run_all']('/usr/sbin/so-bpf-compile ' ~ GLOBALS.sensor.interface ~ ' ' ~ ZEEKBPF|join(" "), cwd='/root') %}
{% set ZEEK_BPF_CALC = salt['cmd.script']('salt://common/tools/sbin/so-bpf-compile', GLOBALS.sensor.interface + ' ' + ZEEKBPF|join(" "),cwd='/root') %}
{% if ZEEK_BPF_CALC['retcode'] == 0 %}
{% set ZEEK_BPF_STATUS = 1 %}
{% endif %}

4
salt/ca/dirs.sls
View File

@@ -1,4 +0,0 @@
pki_issued_certs:
file.directory:
- name: /etc/pki/issued_certs
- makedirs: True

66
salt/ca/init.sls
View File

@@ -3,70 +3,10 @@
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{% from 'allowed_states.map.jinja' import allowed_states %}
{% if sls in allowed_states %}
{% from 'vars/globals.map.jinja' import GLOBALS %}
include:
- ca.dirs
/etc/salt/minion.d/signing_policies.conf:
file.managed:
- source: salt://ca/files/signing_policies.conf
pki_private_key:
x509.private_key_managed:
- name: /etc/pki/ca.key
- keysize: 4096
- passphrase:
- backup: True
{% if salt['file.file_exists']('/etc/pki/ca.key') -%}
- prereq:
- x509: /etc/pki/ca.crt
{%- endif %}
pki_public_ca_crt:
x509.certificate_managed:
- name: /etc/pki/ca.crt
- signing_private_key: /etc/pki/ca.key
- CN: {{ GLOBALS.manager }}
- C: US
- ST: Utah
- L: Salt Lake City
- basicConstraints: "critical CA:true"
- keyUsage: "critical cRLSign, keyCertSign"
- extendedkeyUsage: "serverAuth, clientAuth"
- subjectKeyIdentifier: hash
- authorityKeyIdentifier: keyid:always, issuer
- days_valid: 3650
- days_remaining: 0
- backup: True
- replace: False
- require:
- sls: ca.dirs
- timeout: 30
- retry:
attempts: 5
interval: 30
mine_update_ca_crt:
module.run:
- mine.update: []
- onchanges:
- x509: pki_public_ca_crt
cakeyperms:
file.managed:
- replace: False
- name: /etc/pki/ca.key
- mode: 640
- group: 939
{% else %}
{{sls}}_state_not_allowed:
test.fail_without_changes:
- name: {{sls}}_state_not_allowed
{% if GLOBALS.is_manager %}
- ca.server
{% endif %}
- ca.trustca

3
salt/ca/map.jinja Normal file
View File

@@ -0,0 +1,3 @@
{% set CA = {
'server': pillar.ca.server
}%}

32
salt/ca/remove.sls
View File

@@ -1,7 +1,35 @@
pki_private_key:
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{% set setup_running = salt['cmd.retcode']('pgrep -x so-setup') == 0 %}
{% if setup_running%}
include:
- ssl.remove
remove_pki_private_key:
file.absent:
- name: /etc/pki/ca.key
pki_public_ca_crt:
remove_pki_public_ca_crt:
file.absent:
- name: /etc/pki/ca.crt
remove_trusttheca:
file.absent:
- name: /etc/pki/tls/certs/intca.crt
remove_pki_public_ca_crt_symlink:
file.absent:
- name: /opt/so/saltstack/local/salt/ca/files/ca.crt
{% else %}
so-setup_not_running:
test.show_notification:
- text: "This state is reserved for usage during so-setup."
{% endif %}

63
salt/ca/server.sls Normal file
View File

@@ -0,0 +1,63 @@
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{% from 'allowed_states.map.jinja' import allowed_states %}
{% if sls in allowed_states %}
{% from 'vars/globals.map.jinja' import GLOBALS %}
pki_private_key:
x509.private_key_managed:
- name: /etc/pki/ca.key
- keysize: 4096
- passphrase:
- backup: True
{% if salt['file.file_exists']('/etc/pki/ca.key') -%}
- prereq:
- x509: /etc/pki/ca.crt
{%- endif %}
pki_public_ca_crt:
x509.certificate_managed:
- name: /etc/pki/ca.crt
- signing_private_key: /etc/pki/ca.key
- CN: {{ GLOBALS.manager }}
- C: US
- ST: Utah
- L: Salt Lake City
- basicConstraints: "critical CA:true"
- keyUsage: "critical cRLSign, keyCertSign"
- extendedkeyUsage: "serverAuth, clientAuth"
- subjectKeyIdentifier: hash
- authorityKeyIdentifier: keyid:always, issuer
- days_valid: 3650
- days_remaining: 7
- backup: True
- replace: False
- timeout: 30
- retry:
attempts: 5
interval: 30
pki_public_ca_crt_symlink:
file.symlink:
- name: /opt/so/saltstack/local/salt/ca/files/ca.crt
- target: /etc/pki/ca.crt
- require:
- x509: pki_public_ca_crt
cakeyperms:
file.managed:
- replace: False
- name: /etc/pki/ca.key
- mode: 640
- group: 939
{% else %}
{{sls}}_state_not_allowed:
test.fail_without_changes:
- name: {{sls}}_state_not_allowed
{% endif %}

21
salt/pcap/tools/sbin/so-pcap-export → salt/ca/signing_policy.sls Executable file → Normal file
View File

@@ -1,18 +1,15 @@
#!/bin/bash
#
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
# when the salt-minion signs the cert, a copy is stored here
issued_certs_copypath:
file.directory:
- name: /etc/pki/issued_certs
- makedirs: True
if [ $# -lt 2 ]; then
echo "Usage: $0 <steno-query> Output-Filename"
exit 1
fi
docker exec -t so-sensoroni scripts/stenoquery.sh "$1" -w /nsm/pcapout/$2.pcap
echo ""
echo "If successful, the output was written to: /nsm/pcapout/$2.pcap"
signing_policy:
file.managed:
- name: /etc/salt/minion.d/signing_policies.conf
- source: salt://ca/files/signing_policies.conf

16
salt/pcap/tools/sbin/so-pcap-restart → salt/ca/trustca.sls Executable file → Normal file
View File

@@ -1,12 +1,18 @@
#!/bin/bash
#
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
include:
- docker
# Trust the CA
trusttheca:
file.managed:
- name: /etc/pki/tls/certs/intca.crt
- source: salt://ca/files/ca.crt
- watch_in:
- service: docker_running
- show_changes: False
- makedirs: True
. /usr/sbin/so-common
/usr/sbin/so-restart steno $1

12
salt/common/files/daemon.json
View File

@@ -1,12 +0,0 @@
{
"registry-mirrors": [
"https://:5000"
],
"bip": "172.17.0.1/24",
"default-address-pools": [
{
"base": "172.17.0.0/24",
"size": 24
}
]
}

29
salt/common/init.sls
View File

@@ -20,11 +20,6 @@ kernel.printk:
sysctl.present:
- value: "3 4 1 3"
# Remove variables.txt from /tmp - This is temp
rmvariablesfile:
file.absent:
- name: /tmp/variables.txt
# Add socore Group
socoregroup:
group.present:
@@ -149,35 +144,13 @@ common_sbin_jinja:
- so-import-pcap
{% endif %}
{% if GLOBALS.role == 'so-heavynode' %}
remove_so-pcap-import_heavynode:
file.absent:
- name: /usr/sbin/so-pcap-import
remove_so-import-pcap_heavynode:
file.absent:
- name: /usr/sbin/so-import-pcap
{% endif %}
{% if not GLOBALS.is_manager%}
# prior to 2.4.50 these scripts were in common/tools/sbin on the manager because of soup and distributed to non managers
# these two states remove the scripts from non manager nodes
remove_soup:
file.absent:
- name: /usr/sbin/soup
remove_so-firewall:
file.absent:
- name: /usr/sbin/so-firewall
{% endif %}
so-status_script:
file.managed:
- name: /usr/sbin/so-status
- source: salt://common/tools/sbin/so-status
- mode: 755
{% if GLOBALS.role in GLOBALS.sensor_roles %}
{% if GLOBALS.is_sensor %}
# Add sensor cleanup
so-sensor-clean:
cron.present:

49
salt/common/packages.sls
View File

@@ -1,52 +1,5 @@
# we cannot import GLOBALS from vars/globals.map.jinja in this state since it is called in setup.virt.init
# since it is early in setup of a new VM, the pillars imported in GLOBALS are not yet defined
{% if grains.os_family == 'Debian' %}
commonpkgs:
pkg.installed:
- skip_suggestions: True
- pkgs:
- apache2-utils
- wget
- ntpdate
- jq
- curl
- ca-certificates
- software-properties-common
- apt-transport-https
- openssl
- netcat-openbsd
- sqlite3
- libssl-dev
- procps
- python3-dateutil
- python3-docker
- python3-packaging
- python3-lxml
- git
- rsync
- vim
- tar
- unzip
- bc
{% if grains.oscodename != 'focal' %}
- python3-rich
{% endif %}
{% if grains.oscodename == 'focal' %}
# since Ubuntu requires and internet connection we can use pip to install modules
python3-pip:
pkg.installed
python-rich:
pip.installed:
- name: rich
- target: /usr/local/lib/python3.8/dist-packages/
- require:
- pkg: python3-pip
{% endif %}
{% endif %}
{% if grains.os_family == 'RedHat' %}
remove_mariadb:
pkg.removed:
@@ -84,5 +37,3 @@ commonpkgs:
- unzip
- wget
- yum-utils
{% endif %}

30
salt/common/soup_scripts.sls
View File

@@ -3,8 +3,6 @@
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{% if '2.4' in salt['cp.get_file_str']('/etc/soversion') %}
{% import_yaml '/opt/so/saltstack/local/pillar/global/soc_global.sls' as SOC_GLOBAL %}
{% if SOC_GLOBAL.global.airgap %}
{% set UPDATE_DIR='/tmp/soagupdate/SecurityOnion' %}
@@ -13,14 +11,6 @@
{% endif %}
{% set SOVERSION = salt['file.read']('/etc/soversion').strip() %}
remove_common_soup:
file.absent:
- name: /opt/so/saltstack/default/salt/common/tools/sbin/soup
remove_common_so-firewall:
file.absent:
- name: /opt/so/saltstack/default/salt/common/tools/sbin/so-firewall
# This section is used to put the scripts in place in the Salt file system
# in case a state run tries to overwrite what we do in the next section.
copy_so-common_common_tools_sbin:
@@ -120,23 +110,3 @@ copy_bootstrap-salt_sbin:
- source: {{UPDATE_DIR}}/salt/salt/scripts/bootstrap-salt.sh
- force: True
- preserve: True
{# this is added in 2.4.120 to remove salt repo files pointing to saltproject.io to accomodate the move to broadcom and new bootstrap-salt script #}
{% if salt['pkg.version_cmp'](SOVERSION, '2.4.120') == -1 %}
{% set saltrepofile = '/etc/yum.repos.d/salt.repo' %}
{% if grains.os_family == 'Debian' %}
{% set saltrepofile = '/etc/apt/sources.list.d/salt.list' %}
{% endif %}
remove_saltproject_io_repo_manager:
file.absent:
- name: {{ saltrepofile }}
{% endif %}
{% else %}
fix_23_soup_sbin:
cmd.run:
- name: curl -s -f -o /usr/sbin/soup https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.3/main/salt/common/tools/sbin/soup
fix_23_soup_salt:
cmd.run:
- name: curl -s -f -o /opt/so/saltstack/defalt/salt/common/tools/sbin/soup https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.3/main/salt/common/tools/sbin/soup
{% endif %}

2
salt/common/tools/sbin/so-bpf-compile
View File

@@ -16,7 +16,7 @@
if [ "$#" -lt 2 ]; then
cat 1>&2 <<EOF
$0 compiles a BPF expression to be passed to stenotype to apply a socket filter.
$0 compiles a BPF expression to be passed to PCAP to apply a socket filter.
Its first argument is the interface (link type is required) and all other arguments
are passed to TCPDump.

2
salt/common/tools/sbin/so-checkin
View File

@@ -10,7 +10,7 @@
cat << EOF
so-checkin will run a full salt highstate to apply all salt states. If a highstate is already running, this request will be queued and so it may pause for a few minutes before you see any more output. For more information about so-checkin and salt, please see:
https://docs.securityonion.net/en/2.4/salt.html
https://securityonion.net/docs/salt
EOF

174
salt/common/tools/sbin/so-common
View File

@@ -10,7 +10,7 @@
# and since this same logic is required during installation, it's included in this file.
DEFAULT_SALT_DIR=/opt/so/saltstack/default
DOC_BASE_URL="https://docs.securityonion.net/en/2.4"
DOC_BASE_URL="https://securityonion.net/docs"
if [ -z $NOROOT ]; then
# Check for prerequisites
@@ -333,8 +333,8 @@ get_elastic_agent_vars() {
if [ -f "$defaultsfile" ]; then
ELASTIC_AGENT_TARBALL_VERSION=$(egrep " +version: " $defaultsfile | awk -F: '{print $2}' | tr -d '[:space:]')
ELASTIC_AGENT_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz"
ELASTIC_AGENT_MD5_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5"
ELASTIC_AGENT_URL="https://repo.securityonion.net/file/so-repo/prod/3/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz"
ELASTIC_AGENT_MD5_URL="https://repo.securityonion.net/file/so-repo/prod/3/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5"
ELASTIC_AGENT_FILE="/nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz"
ELASTIC_AGENT_MD5="/nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5"
ELASTIC_AGENT_EXPANSION_DIR=/nsm/elastic-fleet/artifacts/beats/elastic-agent
@@ -349,21 +349,16 @@ get_random_value() {
}
gpg_rpm_import() {
if [[ $is_oracle ]]; then
if [[ "$WHATWOULDYOUSAYYAHDOHERE" == "setup" ]]; then
local RPMKEYSLOC="../salt/repo/client/files/$OS/keys"
else
local RPMKEYSLOC="$UPDATE_DIR/salt/repo/client/files/$OS/keys"
fi
RPMKEYS=('RPM-GPG-KEY-oracle' 'RPM-GPG-KEY-EPEL-9' 'SALT-PROJECT-GPG-PUBKEY-2023.pub' 'docker.pub' 'securityonion.pub')
for RPMKEY in "${RPMKEYS[@]}"; do
rpm --import $RPMKEYSLOC/$RPMKEY
echo "Imported $RPMKEY"
done
elif [[ $is_rpm ]]; then
echo "Importing the security onion GPG key"
rpm --import ../salt/repo/client/files/oracle/keys/securityonion.pub
if [[ "$WHATWOULDYOUSAYYAHDOHERE" == "setup" ]]; then
local RPMKEYSLOC="../salt/repo/client/files/$OS/keys"
else
local RPMKEYSLOC="$UPDATE_DIR/salt/repo/client/files/$OS/keys"
fi
RPMKEYS=('RPM-GPG-KEY-oracle' 'RPM-GPG-KEY-EPEL-9' 'SALT-PROJECT-GPG-PUBKEY-2023.pub' 'docker.pub' 'securityonion.pub')
for RPMKEY in "${RPMKEYS[@]}"; do
rpm --import $RPMKEYSLOC/$RPMKEY
echo "Imported $RPMKEY"
done
}
header() {
@@ -404,6 +399,25 @@ is_single_node_grid() {
grep "role: so-" /etc/salt/grains | grep -E "eval|standalone|import" &> /dev/null
}
initialize_elasticsearch_indices() {
local index_names=$1
local default_entry=${2:-'{"@timestamp":"0"}'}
for idx in $index_names; do
if ! so-elasticsearch-query "$idx" --fail --retry 3 --retry-delay 30 >/dev/null 2>&1; then
echo "Index does not already exist. Initializing $idx index."
if retry 3 10 "so-elasticsearch-query "$idx/_doc" -d '$default_entry' -XPOST --fail 2>/dev/null" '"successful":1'; then
echo "Successfully initialized $idx index."
else
echo "Failed to initialize $idx index after 3 attempts."
fi
else
echo "Index $idx already exists. No action needed."
fi
done
}
lookup_bond_interfaces() {
cat /proc/net/bonding/bond0 | grep "Slave Interface:" | sed -e "s/Slave Interface: //g"
}
@@ -531,6 +545,22 @@ retry() {
return $exitcode
}
rollover_index() {
idx=$1
exists=$(so-elasticsearch-query $idx -o /dev/null -w "%{http_code}")
if [[ $exists -eq 200 ]]; then
rollover=$(so-elasticsearch-query $idx/_rollover -o /dev/null -w "%{http_code}" -XPOST)
if [[ $rollover -eq 200 ]]; then
echo "Successfully triggered rollover for $idx..."
else
echo "Could not trigger rollover for $idx..."
fi
else
echo "Could not find index $idx..."
fi
}
run_check_net_err() {
local cmd=$1
local err_msg=${2:-"Unknown error occured, please check /root/$WHATWOULDYOUSAYYAHDOHERE.log for details."} # Really need to rename that variable
@@ -554,21 +584,39 @@ run_check_net_err() {
}
wait_for_salt_minion() {
local minion="$1"
local timeout="${2:-5}"
local logfile="${3:-'/dev/stdout'}"
retry 60 5 "journalctl -u salt-minion.service | grep 'Minion is ready to receive requests'" >> "$logfile" 2>&1 || fail
local attempt=0
# each attempts would take about 15 seconds
local maxAttempts=20
until check_salt_minion_status "$minion" "$timeout" "$logfile"; do
attempt=$((attempt+1))
if [[ $attempt -eq $maxAttempts ]]; then
return 1
fi
sleep 10
done
return 0
local minion="$1"
local max_wait="${2:-30}"
local interval="${3:-2}"
local logfile="${4:-'/dev/stdout'}"
local elapsed=0
echo "$(date '+%a %d %b %Y %H:%M:%S.%6N') - Waiting for salt-minion '$minion' to be ready..."
while [ $elapsed -lt $max_wait ]; do
# Check if service is running
echo "$(date '+%a %d %b %Y %H:%M:%S.%6N') - Check if salt-minion service is running"
if ! systemctl is-active --quiet salt-minion; then
echo "$(date '+%a %d %b %Y %H:%M:%S.%6N') - salt-minion service not running (elapsed: ${elapsed}s)"
sleep $interval
elapsed=$((elapsed + interval))
continue
fi
echo "$(date '+%a %d %b %Y %H:%M:%S.%6N') - salt-minion service is running"
# Check if minion responds to ping
echo "$(date '+%a %d %b %Y %H:%M:%S.%6N') - Check if $minion responds to ping"
if salt "$minion" test.ping --timeout=3 --out=json 2>> "$logfile" | grep -q "true"; then
echo "$(date '+%a %d %b %Y %H:%M:%S.%6N') - salt-minion '$minion' is connected and ready!"
return 0
fi
echo "$(date '+%a %d %b %Y %H:%M:%S.%6N') - Waiting... (${elapsed}s / ${max_wait}s)"
sleep $interval
elapsed=$((elapsed + interval))
done
echo "$(date '+%a %d %b %Y %H:%M:%S.%6N') - ERROR: salt-minion '$minion' not ready after $max_wait seconds"
return 1
}
salt_minion_count() {
@@ -578,69 +626,19 @@ salt_minion_count() {
}
set_os() {
if [ -f /etc/redhat-release ]; then
if grep -q "Rocky Linux release 9" /etc/redhat-release; then
OS=rocky
OSVER=9
is_rocky=true
is_rpm=true
elif grep -q "CentOS Stream release 9" /etc/redhat-release; then
OS=centos
OSVER=9
is_centos=true
is_rpm=true
elif grep -q "AlmaLinux release 9" /etc/redhat-release; then
OS=alma
OSVER=9
is_alma=true
is_rpm=true
elif grep -q "Red Hat Enterprise Linux release 9" /etc/redhat-release; then
if [ -f /etc/oracle-release ]; then
OS=oracle
OSVER=9
is_oracle=true
is_rpm=true
else
OS=rhel
OSVER=9
is_rhel=true
is_rpm=true
fi
fi
cron_service_name="crond"
elif [ -f /etc/os-release ]; then
if grep -q "UBUNTU_CODENAME=focal" /etc/os-release; then
OSVER=focal
UBVER=20.04
OS=ubuntu
is_ubuntu=true
is_deb=true
elif grep -q "UBUNTU_CODENAME=jammy" /etc/os-release; then
OSVER=jammy
UBVER=22.04
OS=ubuntu
is_ubuntu=true
is_deb=true
elif grep -q "VERSION_CODENAME=bookworm" /etc/os-release; then
OSVER=bookworm
DEBVER=12
is_debian=true
OS=debian
is_deb=true
fi
cron_service_name="cron"
if [ -f /etc/redhat-release ] && grep -q "Red Hat Enterprise Linux release 9" /etc/redhat-release && [ -f /etc/oracle-release ]; then
OS=oracle
OSVER=9
is_oracle=true
is_rpm=true
fi
cron_service_name="crond"
}
set_minionid() {
MINIONID=$(lookup_grain id)
}
set_palette() {
if [[ $is_deb ]]; then
update-alternatives --set newt-palette /etc/newt/palette.original
fi
}
set_version() {
CURRENTVERSION=0.0.0

3
salt/common/tools/sbin/so-image-common
View File

@@ -32,7 +32,6 @@ container_list() {
"so-nginx"
"so-pcaptools"
"so-soc"
"so-steno"
"so-suricata"
"so-telegraf"
"so-zeek"
@@ -58,7 +57,6 @@ container_list() {
"so-pcaptools"
"so-redis"
"so-soc"
"so-steno"
"so-strelka-backend"
"so-strelka-manager"
"so-suricata"
@@ -71,7 +69,6 @@ container_list() {
"so-logstash"
"so-nginx"
"so-redis"
"so-steno"
"so-suricata"
"so-soc"
"so-telegraf"

8
salt/common/tools/sbin/so-log-check
View File

@@ -129,6 +129,9 @@ if [[ $EXCLUDE_STARTUP_ERRORS == 'Y' ]]; then
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|responded with status-code 503" # telegraf getting 503 from ES during startup
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|process_cluster_event_timeout_exception" # logstash waiting for elasticsearch to start
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|not configured for GeoIP" # SO does not bundle the maxminddb with Zeek
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|HTTP 404: Not Found" # Salt loops until Kratos returns 200, during startup Kratos may not be ready
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Cancelling deferred write event maybeFenceReplicas because the event queue is now closed" # Kafka controller log during shutdown/restart
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Redis may have been restarted" # Redis likely restarted by salt
fi
if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then
@@ -159,7 +162,9 @@ if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|adding ingest pipeline" # false positive (elasticsearch ingest pipeline names contain 'error')
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|updating index template" # false positive (elasticsearch index or template names contain 'error')
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|updating component template" # false positive (elasticsearch index or template names contain 'error')
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|upgrading component template" # false positive (elasticsearch index or template names contain 'error')
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|upgrading composable template" # false positive (elasticsearch composable template names contain 'error')
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Error while parsing document for index \[.ds-logs-kratos-so-.*object mapping for \[file\]" # false positive (mapping error occuring BEFORE kratos index has rolled over in 2.4.210)
fi
if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
@@ -175,7 +180,6 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|salt-minion-check" # bug in early 2.4 place Jinja script in non-jinja salt dir causing cron output errors
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|monitoring.metrics" # known issue with elastic agent casting the field incorrectly if an integer value shows up before a float
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|repodownload.conf" # known issue with reposync on pre-2.4.20
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|missing versions record" # stenographer corrupt index
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|soc.field." # known ingest type collisions issue with earlier versions of SO
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|error parsing signature" # Malformed Suricata rule, from upstream provider
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|sticky buffer has no matches" # Non-critical Suricata error
@@ -223,6 +227,8 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|from NIC checksum offloading" # zeek reporter.log
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|marked for removal" # docker container getting recycled
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|tcp 127.0.0.1:6791: bind: address already in use" # so-elastic-fleet agent restarting. Seen starting w/ 8.18.8 https://github.com/elastic/kibana/issues/201459
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|TransformTask\] \[logs-(tychon|aws_billing|microsoft_defender_endpoint).*user so_kibana lacks the required permissions \[logs-\1" # Known issue with 3 integrations using kibana_system role vs creating unique api creds with proper permissions.
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|manifest unknown" # appears in so-dockerregistry log for so-tcpreplay following docker upgrade to 29.2.1-1
fi
RESULT=0

9
salt/common/tools/sbin/so-nsm-clear
View File

@@ -55,19 +55,22 @@ if [ $SKIP -ne 1 ]; then
fi
delete_pcap() {
PCAP_DATA="/nsm/pcap/"
[ -d $PCAP_DATA ] && so-pcap-stop && rm -rf $PCAP_DATA/* && so-pcap-start
PCAP_DATA="/nsm/suripcap/"
[ -d $PCAP_DATA ] && rm -rf $PCAP_DATA/*
}
delete_suricata() {
SURI_LOG="/nsm/suricata/"
[ -d $SURI_LOG ] && so-suricata-stop && rm -rf $SURI_LOG/* && so-suricata-start
[ -d $SURI_LOG ] && rm -rf $SURI_LOG/*
}
delete_zeek() {
ZEEK_LOG="/nsm/zeek/logs/"
[ -d $ZEEK_LOG ] && so-zeek-stop && rm -rf $ZEEK_LOG/* && so-zeek-start
}
so-suricata-stop
delete_pcap
delete_suricata
delete_zeek
so-suricata-start

1
salt/common/tools/sbin/so-restart
View File

@@ -23,7 +23,6 @@ if [ $# -ge 1 ]; then
fi
case $1 in
"steno") docker stop so-steno && docker rm so-steno && salt-call state.apply pcap queue=True;;
"elastic-fleet") docker stop so-elastic-fleet && docker rm so-elastic-fleet && salt-call state.apply elasticfleet queue=True;;
*) docker stop so-$1 ; docker rm so-$1 ; salt-call state.apply $1 queue=True;;
esac

2
salt/common/tools/sbin/so-sensor-clean
View File

@@ -72,7 +72,7 @@ clean() {
done
fi
## Clean up extracted pcaps from Steno
## Clean up extracted pcaps
PCAPS='/nsm/pcapout'
OLDEST_PCAP=$(find $PCAPS -type f -printf '%T+ %p\n' | sort -n | head -n 1)
if [ -z "$OLDEST_PCAP" -o "$OLDEST_PCAP" == ".." -o "$OLDEST_PCAP" == "." ]; then

1
salt/common/tools/sbin/so-start
View File

@@ -23,7 +23,6 @@ if [ $# -ge 1 ]; then
case $1 in
"all") salt-call state.highstate queue=True;;
"steno") if docker ps | grep -q so-$1; then printf "\n$1 is already running!\n\n"; else docker rm so-$1 >/dev/null 2>&1 ; salt-call state.apply pcap queue=True; fi ;;
"elastic-fleet") if docker ps | grep -q so-$1; then printf "\n$1 is already running!\n\n"; else docker rm so-$1 >/dev/null 2>&1 ; salt-call state.apply elasticfleet queue=True; fi ;;
*) if docker ps | grep -E -q '^so-$1$'; then printf "\n$1 is already running\n\n"; else docker rm so-$1 >/dev/null 2>&1 ; salt-call state.apply $1 queue=True; fi ;;
esac

2
salt/common/tools/sbin_jinja/so-desktop-install
View File

@@ -6,7 +6,7 @@
# Elastic License 2.0.
source /usr/sbin/so-common
doc_desktop_url="$DOC_BASE_URL/desktop.html"
doc_desktop_url="$DOC_BASE_URL/desktop"
{# we only want the script to install the desktop if it is OEL -#}
{% if grains.os == 'OEL' -%}

34
salt/curator/disabled.sls
View File

@@ -1,34 +0,0 @@
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
so-curator:
docker_container.absent:
- force: True
so-curator_so-status.disabled:
file.line:
- name: /opt/so/conf/so-status/so-status.conf
- match: ^so-curator$
- mode: delete
so-curator-cluster-close:
cron.absent:
- identifier: so-curator-cluster-close
so-curator-cluster-delete:
cron.absent:
- identifier: so-curator-cluster-delete
delete_curator_configuration:
file.absent:
- name: /opt/so/conf/curator
- recurse: True
{% set files = salt.file.find(path='/usr/sbin', name='so-curator*') %}
{% if files|length > 0 %}
delete_curator_scripts:
file.absent:
- names: {{files|yaml}}
{% endif %}

19
salt/desktop/trusted-ca.sls
View File

@@ -3,29 +3,16 @@
{# we only want this state to run it is CentOS #}
{% if GLOBALS.os == 'OEL' %}
{% set global_ca_text = [] %}
{% set global_ca_server = [] %}
{% set manager = GLOBALS.manager %}
{% set x509dict = salt['mine.get'](manager | lower~'*', 'x509.get_pem_entries') %}
{% for host in x509dict %}
{% if host.split('_')|last in ['manager', 'managersearch', 'standalone', 'import', 'eval'] %}
{% do global_ca_text.append(x509dict[host].get('/etc/pki/ca.crt')|replace('\n', '')) %}
{% do global_ca_server.append(host) %}
{% endif %}
{% endfor %}
{% set trusttheca_text = global_ca_text[0] %}
{% set ca_server = global_ca_server[0] %}
trusted_ca:
x509.pem_managed:
file.managed:
- name: /etc/pki/ca-trust/source/anchors/ca.crt
- text: {{ trusttheca_text }}
- source: salt://ca/files/ca.crt
update_ca_certs:
cmd.run:
- name: update-ca-trust
- onchanges:
- x509: trusted_ca
- file: trusted_ca
{% else %}

50
salt/docker/defaults.yaml
View File

@@ -1,6 +1,10 @@
docker:
range: '172.17.1.0/24'
gateway: '172.17.1.1'
ulimits:
- name: nofile
soft: 1048576
hard: 1048576
containers:
'so-dockerregistry':
final_octet: 20
@@ -9,6 +13,7 @@ docker:
custom_bind_mounts: []
extra_hosts: []
extra_env: []
ulimits: []
'so-elastic-fleet':
final_octet: 21
port_bindings:
@@ -16,6 +21,7 @@ docker:
custom_bind_mounts: []
extra_hosts: []
extra_env: []
ulimits: []
'so-elasticsearch':
final_octet: 22
port_bindings:
@@ -24,6 +30,16 @@ docker:
custom_bind_mounts: []
extra_hosts: []
extra_env: []
ulimits:
- name: memlock
soft: -1
hard: -1
- name: nofile
soft: 65536
hard: 65536
- name: nproc
soft: 4096
hard: 4096
'so-influxdb':
final_octet: 26
port_bindings:
@@ -31,6 +47,7 @@ docker:
custom_bind_mounts: []
extra_hosts: []
extra_env: []
ulimits: []
'so-kibana':
final_octet: 27
port_bindings:
@@ -38,6 +55,7 @@ docker:
custom_bind_mounts: []
extra_hosts: []
extra_env: []
ulimits: []
'so-kratos':
final_octet: 28
port_bindings:
@@ -46,6 +64,7 @@ docker:
custom_bind_mounts: []
extra_hosts: []
extra_env: []
ulimits: []
'so-hydra':
final_octet: 30
port_bindings:
@@ -54,6 +73,7 @@ docker:
custom_bind_mounts: []
extra_hosts: []
extra_env: []
ulimits: []
'so-logstash':
final_octet: 29
port_bindings:
@@ -70,6 +90,7 @@ docker:
custom_bind_mounts: []
extra_hosts: []
extra_env: []
ulimits: []
'so-nginx':
final_octet: 31
port_bindings:
@@ -81,6 +102,7 @@ docker:
custom_bind_mounts: []
extra_hosts: []
extra_env: []
ulimits: []
'so-nginx-fleet-node':
final_octet: 31
port_bindings:
@@ -88,6 +110,7 @@ docker:
custom_bind_mounts: []
extra_hosts: []
extra_env: []
ulimits: []
'so-redis':
final_octet: 33
port_bindings:
@@ -96,11 +119,13 @@ docker:
custom_bind_mounts: []
extra_hosts: []
extra_env: []
ulimits: []
'so-sensoroni':
final_octet: 99
custom_bind_mounts: []
extra_hosts: []
extra_env: []
ulimits: []
'so-soc':
final_octet: 34
port_bindings:
@@ -108,16 +133,19 @@ docker:
custom_bind_mounts: []
extra_hosts: []
extra_env: []
ulimits: []
'so-strelka-backend':
final_octet: 36
custom_bind_mounts: []
extra_hosts: []
extra_env: []
ulimits: []
'so-strelka-filestream':
final_octet: 37
custom_bind_mounts: []
extra_hosts: []
extra_env: []
ulimits: []
'so-strelka-frontend':
final_octet: 38
port_bindings:
@@ -125,11 +153,13 @@ docker:
custom_bind_mounts: []
extra_hosts: []
extra_env: []
ulimits: []
'so-strelka-manager':
final_octet: 39
custom_bind_mounts: []
extra_hosts: []
extra_env: []
ulimits: []
'so-strelka-gatekeeper':
final_octet: 40
port_bindings:
@@ -137,6 +167,7 @@ docker:
custom_bind_mounts: []
extra_hosts: []
extra_env: []
ulimits: []
'so-strelka-coordinator':
final_octet: 41
port_bindings:
@@ -144,11 +175,13 @@ docker:
custom_bind_mounts: []
extra_hosts: []
extra_env: []
ulimits: []
'so-elastalert':
final_octet: 42
custom_bind_mounts: []
extra_hosts: []
extra_env: []
ulimits: []
'so-elastic-fleet-package-registry':
final_octet: 44
port_bindings:
@@ -156,11 +189,13 @@ docker:
custom_bind_mounts: []
extra_hosts: []
extra_env: []
ulimits: []
'so-idh':
final_octet: 45
custom_bind_mounts: []
extra_hosts: []
extra_env: []
ulimits: []
'so-elastic-agent':
final_octet: 46
port_bindings:
@@ -169,28 +204,28 @@ docker:
custom_bind_mounts: []
extra_hosts: []
extra_env: []
ulimits: []
'so-telegraf':
final_octet: 99
custom_bind_mounts: []
extra_hosts: []
extra_env: []
'so-steno':
final_octet: 99
custom_bind_mounts: []
extra_hosts: []
extra_env: []
ulimits: []
'so-suricata':
final_octet: 99
custom_bind_mounts: []
extra_hosts: []
extra_env: []
ulimits:
- memlock=524288000
ulimits: []
'so-zeek':
final_octet: 99
custom_bind_mounts: []
extra_hosts: []
extra_env: []
ulimits:
- name: core
soft: 0
hard: 0
'so-kafka':
final_octet: 88
port_bindings:
@@ -201,3 +236,4 @@ docker:
custom_bind_mounts: []
extra_hosts: []
extra_env: []
ulimits: []

8
salt/docker/docker.map.jinja
View File

@@ -1,8 +1,8 @@
{% import_yaml 'docker/defaults.yaml' as DOCKERDEFAULTS %}
{% set DOCKER = salt['pillar.get']('docker', DOCKERDEFAULTS.docker, merge=True) %}
{% set RANGESPLIT = DOCKER.range.split('.') %}
{% set DOCKERMERGED = salt['pillar.get']('docker', DOCKERDEFAULTS.docker, merge=True) %}
{% set RANGESPLIT = DOCKERMERGED.range.split('.') %}
{% set FIRSTTHREE = RANGESPLIT[0] ~ '.' ~ RANGESPLIT[1] ~ '.' ~ RANGESPLIT[2] ~ '.' %}
{% for container, vals in DOCKER.containers.items() %}
{% do DOCKER.containers[container].update({'ip': FIRSTTHREE ~ DOCKER.containers[container].final_octet}) %}
{% for container, vals in DOCKERMERGED.containers.items() %}
{% do DOCKERMERGED.containers[container].update({'ip': FIRSTTHREE ~ DOCKERMERGED.containers[container].final_octet}) %}
{% endfor %}

24
salt/docker/files/daemon.json.jinja Normal file
View File

@@ -0,0 +1,24 @@
{% from 'docker/docker.map.jinja' import DOCKERMERGED -%}
{
"registry-mirrors": [
"https://:5000"
],
"bip": "172.17.0.1/24",
"default-address-pools": [
{
"base": "172.17.0.0/24",
"size": 24
}
]
{%- if DOCKERMERGED.ulimits %},
"default-ulimits": {
{%- for ULIMIT in DOCKERMERGED.ulimits %}
"{{ ULIMIT.name }}": {
"Name": "{{ ULIMIT.name }}",
"Soft": {{ ULIMIT.soft }},
"Hard": {{ ULIMIT.hard }}
}{{ "," if not loop.last else "" }}
{%- endfor %}
}
{%- endif %}
}

60
salt/docker/init.sls
View File

@@ -3,61 +3,27 @@
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{% from 'docker/docker.map.jinja' import DOCKER %}
{% from 'docker/docker.map.jinja' import DOCKERMERGED %}
{% from 'vars/globals.map.jinja' import GLOBALS %}
# include ssl since docker service requires the intca
# docker service requires the ca.crt
include:
- ssl
- ca
dockergroup:
group.present:
- name: docker
- gid: 920
{% if GLOBALS.os_family == 'Debian' %}
{% if grains.oscodename == 'bookworm' %}
dockerheldpackages:
pkg.installed:
- pkgs:
- containerd.io: 1.7.21-1
- docker-ce: 5:27.2.0-1~debian.12~bookworm
- docker-ce-cli: 5:27.2.0-1~debian.12~bookworm
- docker-ce-rootless-extras: 5:27.2.0-1~debian.12~bookworm
- containerd.io: 2.2.1-1.el9
- docker-ce: 3:29.2.1-1.el9
- docker-ce-cli: 1:29.2.1-1.el9
- docker-ce-rootless-extras: 29.2.1-1.el9
- hold: True
- update_holds: True
{% elif grains.oscodename == 'jammy' %}
dockerheldpackages:
pkg.installed:
- pkgs:
- containerd.io: 1.7.21-1
- docker-ce: 5:27.2.0-1~ubuntu.22.04~jammy
- docker-ce-cli: 5:27.2.0-1~ubuntu.22.04~jammy
- docker-ce-rootless-extras: 5:27.2.0-1~ubuntu.22.04~jammy
- hold: True
- update_holds: True
{% else %}
dockerheldpackages:
pkg.installed:
- pkgs:
- containerd.io: 1.7.21-1
- docker-ce: 5:27.2.0-1~ubuntu.20.04~focal
- docker-ce-cli: 5:27.2.0-1~ubuntu.20.04~focal
- docker-ce-rootless-extras: 5:27.2.0-1~ubuntu.20.04~focal
- hold: True
- update_holds: True
{% endif %}
{% else %}
dockerheldpackages:
pkg.installed:
- pkgs:
- containerd.io: 1.7.21-3.1.el9
- docker-ce: 3:27.2.0-1.el9
- docker-ce-cli: 1:27.2.0-1.el9
- docker-ce-rootless-extras: 27.2.0-1.el9
- hold: True
- update_holds: True
{% endif %}
#disable docker from managing iptables
iptables_disabled:
@@ -75,10 +41,9 @@ dockeretc:
file.directory:
- name: /etc/docker
# Manager daemon.json
docker_daemon:
file.managed:
- source: salt://common/files/daemon.json
- source: salt://docker/files/daemon.json.jinja
- name: /etc/docker/daemon.json
- template: jinja
@@ -89,10 +54,9 @@ docker_running:
- enable: True
- watch:
- file: docker_daemon
- x509: trusttheca
- require:
- file: docker_daemon
- x509: trusttheca
- file: trusttheca
# Reserve OS ports for Docker proxy in case boot settings are not already applied/present
@@ -110,12 +74,12 @@ dockerreserveports:
sos_docker_net:
docker_network.present:
- name: sobridge
- subnet: {{ DOCKER.range }}
- gateway: {{ DOCKER.gateway }}
- subnet: {{ DOCKERMERGED.range }}
- gateway: {{ DOCKERMERGED.gateway }}
- options:
com.docker.network.bridge.name: 'sobridge'
com.docker.network.driver.mtu: '1500'
com.docker.network.bridge.enable_ip_masquerade: 'true'
com.docker.network.bridge.enable_icc: 'true'
com.docker.network.bridge.host_binding_ipv4: '0.0.0.0'
- unless: 'docker network ls | grep sobridge'
- unless: ip l | grep sobridge

91
salt/docker/soc_docker.yaml
View File

@@ -1,44 +1,82 @@
docker:
gateway:
description: Gateway for the default docker interface.
helpLink: docker.html
helpLink: docker
advanced: True
range:
description: Default docker IP range for containers.
helpLink: docker.html
helpLink: docker
advanced: True
ulimits:
description: |
Default ulimit settings applied to all containers via the Docker daemon. Each entry specifies a resource name (e.g. nofile, memlock, core, nproc) with soft and hard limits. Individual container ulimits override these defaults. Valid resource names include: cpu, fsize, data, stack, core, rss, nproc, nofile, memlock, as, locks, sigpending, msgqueue, nice, rtprio, rttime.
forcedType: "[]{}"
syntax: json
advanced: True
helpLink: docker.html
uiElements:
- field: name
label: Resource Name
required: True
regex: ^(cpu|fsize|data|stack|core|rss|nproc|nofile|memlock|as|locks|sigpending|msgqueue|nice|rtprio|rttime)$
regexFailureMessage: You must enter a valid ulimit name (cpu, fsize, data, stack, core, rss, nproc, nofile, memlock, as, locks, sigpending, msgqueue, nice, rtprio, rttime).
- field: soft
label: Soft Limit
forcedType: int
- field: hard
label: Hard Limit
forcedType: int
containers:
so-dockerregistry: &dockerOptions
final_octet:
description: Last octet of the container IP address.
helpLink: docker.html
helpLink: docker
readonly: True
advanced: True
global: True
port_bindings:
description: List of port bindings for the container.
helpLink: docker.html
helpLink: docker
advanced: True
multiline: True
forcedType: "[]string"
custom_bind_mounts:
description: List of custom local volume bindings.
advanced: True
helpLink: docker.html
helpLink: docker
multiline: True
forcedType: "[]string"
extra_hosts:
description: List of additional host entries for the container.
advanced: True
helpLink: docker.html
helpLink: docker
multiline: True
forcedType: "[]string"
extra_env:
description: List of additional ENV entries for the container.
advanced: True
helpLink: docker.html
helpLink: docker
multiline: True
forcedType: "[]string"
ulimits:
description: |
Ulimit settings for the container. Each entry specifies a resource name (e.g. nofile, memlock, core, nproc) with optional soft and hard limits. Valid resource names include: cpu, fsize, data, stack, core, rss, nproc, nofile, memlock, as, locks, sigpending, msgqueue, nice, rtprio, rttime.
advanced: True
helpLink: docker.html
forcedType: "[]{}"
syntax: json
uiElements:
- field: name
label: Resource Name
required: True
regex: ^(cpu|fsize|data|stack|core|rss|nproc|nofile|memlock|as|locks|sigpending|msgqueue|nice|rtprio|rttime)$
regexFailureMessage: You must enter a valid ulimit name (cpu, fsize, data, stack, core, rss, nproc, nofile, memlock, as, locks, sigpending, msgqueue, nice, rtprio, rttime).
- field: soft
label: Soft Limit
forcedType: int
- field: hard
label: Hard Limit
forcedType: int
so-elastic-fleet: *dockerOptions
so-elasticsearch: *dockerOptions
so-influxdb: *dockerOptions
@@ -62,43 +100,6 @@ docker:
so-idh: *dockerOptions
so-elastic-agent: *dockerOptions
so-telegraf: *dockerOptions
so-steno: *dockerOptions
so-suricata:
final_octet:
description: Last octet of the container IP address.
helpLink: docker.html
readonly: True
advanced: True
global: True
port_bindings:
description: List of port bindings for the container.
helpLink: docker.html
advanced: True
multiline: True
forcedType: "[]string"
custom_bind_mounts:
description: List of custom local volume bindings.
advanced: True
helpLink: docker.html
multiline: True
forcedType: "[]string"
extra_hosts:
description: List of additional host entries for the container.
advanced: True
helpLink: docker.html
multiline: True
forcedType: "[]string"
extra_env:
description: List of additional ENV entries for the container.
advanced: True
helpLink: docker.html
multiline: True
forcedType: "[]string"
ulimits:
description: Ulimits for the container, in bytes.
advanced: True
helpLink: docker.html
multiline: True
forcedType: "[]string"
so-suricata: *dockerOptions
so-zeek: *dockerOptions
so-kafka: *dockerOptions

24
salt/elastalert/enabled.sls
View File

@@ -6,7 +6,7 @@
{% from 'allowed_states.map.jinja' import allowed_states %}
{% if sls.split('.')[0] in allowed_states %}
{% from 'vars/globals.map.jinja' import GLOBALS %}
{% from 'docker/docker.map.jinja' import DOCKER %}
{% from 'docker/docker.map.jinja' import DOCKERMERGED %}
include:
- elastalert.config
@@ -24,7 +24,7 @@ so-elastalert:
- user: so-elastalert
- networks:
- sobridge:
- ipv4_address: {{ DOCKER.containers['so-elastalert'].ip }}
- ipv4_address: {{ DOCKERMERGED.containers['so-elastalert'].ip }}
- detach: True
- binds:
- /opt/so/rules/elastalert:/opt/elastalert/rules/:ro
@@ -33,24 +33,30 @@ so-elastalert:
- /opt/so/conf/elastalert/predefined/:/opt/elastalert/predefined/:ro
- /opt/so/conf/elastalert/custom/:/opt/elastalert/custom/:ro
- /opt/so/conf/elastalert/elastalert_config.yaml:/opt/elastalert/config.yaml:ro
{% if DOCKER.containers['so-elastalert'].custom_bind_mounts %}
{% for BIND in DOCKER.containers['so-elastalert'].custom_bind_mounts %}
{% if DOCKERMERGED.containers['so-elastalert'].custom_bind_mounts %}
{% for BIND in DOCKERMERGED.containers['so-elastalert'].custom_bind_mounts %}
- {{ BIND }}
{% endfor %}
{% endif %}
- extra_hosts:
- {{ GLOBALS.manager }}:{{ GLOBALS.manager_ip }}
{% if DOCKER.containers['so-elastalert'].extra_hosts %}
{% for XTRAHOST in DOCKER.containers['so-elastalert'].extra_hosts %}
{% if DOCKERMERGED.containers['so-elastalert'].extra_hosts %}
{% for XTRAHOST in DOCKERMERGED.containers['so-elastalert'].extra_hosts %}
- {{ XTRAHOST }}
{% endfor %}
{% endif %}
{% if DOCKER.containers['so-elastalert'].extra_env %}
{% if DOCKERMERGED.containers['so-elastalert'].extra_env %}
- environment:
{% for XTRAENV in DOCKER.containers['so-elastalert'].extra_env %}
{% for XTRAENV in DOCKERMERGED.containers['so-elastalert'].extra_env %}
- {{ XTRAENV }}
{% endfor %}
{% endif %}
{% if DOCKERMERGED.containers['so-elastalert'].ulimits %}
- ulimits:
{% for ULIMIT in DOCKERMERGED.containers['so-elastalert'].ulimits %}
- {{ ULIMIT.name }}={{ ULIMIT.soft }}:{{ ULIMIT.hard }}
{% endfor %}
{% endif %}
- require:
- cmd: wait_for_elasticsearch
- file: elastarules
@@ -60,7 +66,7 @@ so-elastalert:
- watch:
- file: elastaconf
- onlyif:
- "so-elasticsearch-query / | jq -r '.version.number[0:1]' | grep -q 8" {# only run this state if elasticsearch is version 8 #}
- "so-elasticsearch-query / | jq -r '.version.number[0:1]' | grep -q 9" {# only run this state if elasticsearch is version 9 #}
delete_so-elastalert_so-status.disabled:
file.uncomment:

0
salt/elasticfleet/files/certs/placeholder → salt/elastalert/files/custom/.gitkeep
View File

1
salt/elastalert/files/custom/placeholder
View File

@@ -1 +0,0 @@
THIS IS A PLACEHOLDER FILE

93
salt/elastalert/soc_elastalert.yaml
View File

@@ -1,47 +1,48 @@
elastalert:
enabled:
description: Enables or disables the ElastAlert 2 process. This process is critical for ensuring alerts arrive in SOC, and for outbound notification delivery.
helpLink: elastalert.html
forcedType: bool
helpLink: elastalert
alerter_parameters:
title: Custom Configuration Parameters
description: Optional configuration parameters made available as defaults for all rules and alerters. Use YAML format for these parameters, and reference the ElastAlert 2 documentation, located at https://elastalert2.readthedocs.io, for available configuration parameters. Requires a valid Security Onion license key.
global: True
multiline: True
syntax: yaml
helpLink: elastalert.html
helpLink: elastalert
forcedType: string
jira_api_key:
title: Jira API Key
description: Optional configuration parameter for Jira API Key, used instead of the Jira username and password. Requires a valid Security Onion license key.
global: True
sensitive: True
helpLink: elastalert.html
helpLink: elastalert
forcedType: string
jira_pass:
title: Jira Password
description: Optional configuration parameter for Jira password. Requires a valid Security Onion license key.
global: True
sensitive: True
helpLink: elastalert.html
helpLink: elastalert
forcedType: string
jira_user:
title: Jira Username
description: Optional configuration parameter for Jira username. Requires a valid Security Onion license key.
global: True
helpLink: elastalert.html
helpLink: elastalert
forcedType: string
smtp_pass:
title: SMTP Password
description: Optional configuration parameter for SMTP password, required for authenticating email servers. Requires a valid Security Onion license key.
global: True
sensitive: True
helpLink: elastalert.html
helpLink: elastalert
forcedType: string
smtp_user:
title: SMTP Username
description: Optional configuration parameter for SMTP username, required for authenticating email servers. Requires a valid Security Onion license key.
global: True
helpLink: elastalert.html
helpLink: elastalert
forcedType: string
files:
custom:
@@ -49,91 +50,131 @@ elastalert:
description: Optional custom Certificate Authority for connecting to an AlertManager server. To utilize this custom file, the alertmanager_ca_certs key must be set to /opt/elastalert/custom/alertmanager_ca.crt in the Alerter Parameters setting. Requires a valid Security Onion license key.
global: True
file: True
helpLink: elastalert.html
helpLink: elastalert
gelf_ca__crt:
description: Optional custom Certificate Authority for connecting to a Graylog server. To utilize this custom file, the graylog_ca_certs key must be set to /opt/elastalert/custom/graylog_ca.crt in the Alerter Parameters setting. Requires a valid Security Onion license key.
global: True
file: True
helpLink: elastalert.html
helpLink: elastalert
http_post_ca__crt:
description: Optional custom Certificate Authority for connecting to a generic HTTP server, via the legacy HTTP POST alerter. To utilize this custom file, the http_post_ca_certs key must be set to /opt/elastalert/custom/http_post2_ca.crt in the Alerter Parameters setting. Requires a valid Security Onion license key.
global: True
file: True
helpLink: elastalert.html
helpLink: elastalert
http_post2_ca__crt:
description: Optional custom Certificate Authority for connecting to a generic HTTP server, via the newer HTTP POST 2 alerter. To utilize this custom file, the http_post2_ca_certs key must be set to /opt/elastalert/custom/http_post2_ca.crt in the Alerter Parameters setting. Requires a valid Security Onion license key.
global: True
file: True
helpLink: elastalert.html
helpLink: elastalert
ms_teams_ca__crt:
description: Optional custom Certificate Authority for connecting to Microsoft Teams server. To utilize this custom file, the ms_teams_ca_certs key must be set to /opt/elastalert/custom/ms_teams_ca.crt in the Alerter Parameters setting. Requires a valid Security Onion license key.
global: True
file: True
helpLink: elastalert.html
helpLink: elastalert
pagerduty_ca__crt:
description: Optional custom Certificate Authority for connecting to PagerDuty server. To utilize this custom file, the pagerduty_ca_certs key must be set to /opt/elastalert/custom/pagerduty_ca.crt in the Alerter Parameters setting. Requires a valid Security Onion license key.
global: True
file: True
helpLink: elastalert.html
helpLink: elastalert
rocket_chat_ca__crt:
description: Optional custom Certificate Authority for connecting to PagerDuty server. To utilize this custom file, the rocket_chart_ca_certs key must be set to /opt/elastalert/custom/rocket_chat_ca.crt in the Alerter Parameters setting. Requires a valid Security Onion license key.
global: True
file: True
helpLink: elastalert.html
helpLink: elastalert
smtp__crt:
description: Optional custom certificate for connecting to an SMTP server. To utilize this custom file, the smtp_cert_file key must be set to /opt/elastalert/custom/smtp.crt in the Alerter Parameters setting. Requires a valid Security Onion license key.
global: True
file: True
helpLink: elastalert.html
helpLink: elastalert
smtp__key:
description: Optional custom certificate key for connecting to an SMTP server. To utilize this custom file, the smtp_key_file key must be set to /opt/elastalert/custom/smtp.key in the Alerter Parameters setting. Requires a valid Security Onion license key.
global: True
file: True
helpLink: elastalert.html
helpLink: elastalert
slack_ca__crt:
description: Optional custom Certificate Authority for connecting to Slack. To utilize this custom file, the slack_ca_certs key must be set to /opt/elastalert/custom/slack_ca.crt in the Alerter Parameters setting. Requires a valid Security Onion license key.
global: True
file: True
helpLink: elastalert.html
helpLink: elastalert
config:
scan_subdirectories:
description: Recursively scan subdirectories for rules.
forcedType: bool
advanced: True
global: True
helpLink: elastalert
disable_rules_on_error:
description: Disable rules on failure.
forcedType: bool
global: True
helpLink: elastalert.html
helpLink: elastalert
run_every:
minutes:
description: Amount of time in minutes between searches.
global: True
helpLink: elastalert.html
helpLink: elastalert
buffer_time:
minutes:
description: Amount of time in minutes to look through.
global: True
helpLink: elastalert.html
helpLink: elastalert
old_query_limit:
minutes:
description: Amount of time in minutes between queries to start at the most recently run query.
global: True
helpLink: elastalert.html
helpLink: elastalert
es_conn_timeout:
description: Timeout in seconds for connecting to and reading from Elasticsearch.
global: True
helpLink: elastalert.html
helpLink: elastalert
max_query_size:
description: The maximum number of documents that will be returned from Elasticsearch in a single query.
global: True
helpLink: elastalert.html
helpLink: elastalert
use_ssl:
description: Use SSL to connect to Elasticsearch.
forcedType: bool
advanced: True
global: True
helpLink: elastalert
verify_certs:
description: Verify TLS certificates when connecting to Elasticsearch.
forcedType: bool
advanced: True
global: True
helpLink: elastalert
alert_time_limit:
days:
description: The retry window for failed alerts.
global: True
helpLink: elastalert.html
helpLink: elastalert
index_settings:
shards:
description: The number of shards for elastalert indices.
global: True
helpLink: elastalert.html
helpLink: elastalert
replicas:
description: The number of replicas for elastalert indices.
global: True
helpLink: elastalert.html
helpLink: elastalert
logging:
incremental:
description: When incremental is false (the default), the logging configuration is applied in full, replacing any existing logging setup. When true, only the level attributes of existing loggers and handlers are updated, leaving the rest of the logging configuration unchanged.
forcedType: bool
advanced: True
global: True
helpLink: elastalert
disable_existing_loggers:
description: Disable existing loggers.
forcedType: bool
advanced: True
global: True
helpLink: elastalert
loggers:
'':
propagate:
description: Propagate log messages to parent loggers.
forcedType: bool
advanced: True
global: True
helpLink: elastalert

24
salt/elastic-fleet-package-registry/enabled.sls
View File

@@ -6,7 +6,7 @@
{% from 'allowed_states.map.jinja' import allowed_states %}
{% if sls.split('.')[0] in allowed_states %}
{% from 'vars/globals.map.jinja' import GLOBALS %}
{% from 'docker/docker.map.jinja' import DOCKER %}
{% from 'docker/docker.map.jinja' import DOCKERMERGED %}
include:
- elastic-fleet-package-registry.config
@@ -21,30 +21,36 @@ so-elastic-fleet-package-registry:
- user: 948
- networks:
- sobridge:
- ipv4_address: {{ DOCKER.containers['so-elastic-fleet-package-registry'].ip }}
- ipv4_address: {{ DOCKERMERGED.containers['so-elastic-fleet-package-registry'].ip }}
- extra_hosts:
- {{ GLOBALS.hostname }}:{{ GLOBALS.node_ip }}
{% if DOCKER.containers['so-elastic-fleet-package-registry'].extra_hosts %}
{% for XTRAHOST in DOCKER.containers['so-elastic-fleet-package-registry'].extra_hosts %}
{% if DOCKERMERGED.containers['so-elastic-fleet-package-registry'].extra_hosts %}
{% for XTRAHOST in DOCKERMERGED.containers['so-elastic-fleet-package-registry'].extra_hosts %}
- {{ XTRAHOST }}
{% endfor %}
{% endif %}
- port_bindings:
{% for BINDING in DOCKER.containers['so-elastic-fleet-package-registry'].port_bindings %}
{% for BINDING in DOCKERMERGED.containers['so-elastic-fleet-package-registry'].port_bindings %}
- {{ BINDING }}
{% endfor %}
{% if DOCKER.containers['so-elastic-fleet-package-registry'].custom_bind_mounts %}
{% if DOCKERMERGED.containers['so-elastic-fleet-package-registry'].custom_bind_mounts %}
- binds:
{% for BIND in DOCKER.containers['so-elastic-fleet-package-registry'].custom_bind_mounts %}
{% for BIND in DOCKERMERGED.containers['so-elastic-fleet-package-registry'].custom_bind_mounts %}
- {{ BIND }}
{% endfor %}
{% endif %}
{% if DOCKER.containers['so-elastic-fleet-package-registry'].extra_env %}
{% if DOCKERMERGED.containers['so-elastic-fleet-package-registry'].extra_env %}
- environment:
{% for XTRAENV in DOCKER.containers['so-elastic-fleet-package-registry'].extra_env %}
{% for XTRAENV in DOCKERMERGED.containers['so-elastic-fleet-package-registry'].extra_env %}
- {{ XTRAENV }}
{% endfor %}
{% endif %}
{% if DOCKERMERGED.containers['so-elastic-fleet-package-registry'].ulimits %}
- ulimits:
{% for ULIMIT in DOCKERMERGED.containers['so-elastic-fleet-package-registry'].ulimits %}
- {{ ULIMIT.name }}={{ ULIMIT.soft }}:{{ ULIMIT.hard }}
{% endfor %}
{% endif %}
delete_so-elastic-fleet-package-registry_so-status.disabled:
file.uncomment:
- name: /opt/so/conf/so-status/so-status.conf

1
salt/elastic-fleet-package-registry/soc_elastic-fleet-package-registry.yaml
View File

@@ -1,4 +1,5 @@
elastic_fleet_package_registry:
enabled:
description: Enables or disables the Fleet package registry process. This process must remain enabled to allow Elastic Agent packages to be updated.
forcedType: bool
advanced: True

27
salt/elasticagent/enabled.sls
View File

@@ -6,9 +6,10 @@
{% from 'allowed_states.map.jinja' import allowed_states %}
{% if sls.split('.')[0] in allowed_states %}
{% from 'vars/globals.map.jinja' import GLOBALS %}
{% from 'docker/docker.map.jinja' import DOCKER %}
{% from 'docker/docker.map.jinja' import DOCKERMERGED %}
include:
- ca
- elasticagent.config
- elasticagent.sostatus
@@ -21,17 +22,17 @@ so-elastic-agent:
- user: 949
- networks:
- sobridge:
- ipv4_address: {{ DOCKER.containers['so-elastic-agent'].ip }}
- ipv4_address: {{ DOCKERMERGED.containers['so-elastic-agent'].ip }}
- extra_hosts:
- {{ GLOBALS.manager }}:{{ GLOBALS.manager_ip }}
- {{ GLOBALS.hostname }}:{{ GLOBALS.node_ip }}
{% if DOCKER.containers['so-elastic-agent'].extra_hosts %}
{% for XTRAHOST in DOCKER.containers['so-elastic-agent'].extra_hosts %}
{% if DOCKERMERGED.containers['so-elastic-agent'].extra_hosts %}
{% for XTRAHOST in DOCKERMERGED.containers['so-elastic-agent'].extra_hosts %}
- {{ XTRAHOST }}
{% endfor %}
{% endif %}
- port_bindings:
{% for BINDING in DOCKER.containers['so-elastic-agent'].port_bindings %}
{% for BINDING in DOCKERMERGED.containers['so-elastic-agent'].port_bindings %}
- {{ BINDING }}
{% endfor %}
- binds:
@@ -40,23 +41,31 @@ so-elastic-agent:
- /etc/pki/tls/certs/intca.crt:/etc/pki/tls/certs/intca.crt:ro
- /nsm:/nsm:ro
- /opt/so/log:/opt/so/log:ro
{% if DOCKER.containers['so-elastic-agent'].custom_bind_mounts %}
{% for BIND in DOCKER.containers['so-elastic-agent'].custom_bind_mounts %}
{% if DOCKERMERGED.containers['so-elastic-agent'].custom_bind_mounts %}
{% for BIND in DOCKERMERGED.containers['so-elastic-agent'].custom_bind_mounts %}
- {{ BIND }}
{% endfor %}
{% endif %}
- environment:
- FLEET_CA=/etc/pki/tls/certs/intca.crt
- LOGS_PATH=logs
{% if DOCKER.containers['so-elastic-agent'].extra_env %}
{% for XTRAENV in DOCKER.containers['so-elastic-agent'].extra_env %}
{% if DOCKERMERGED.containers['so-elastic-agent'].extra_env %}
{% for XTRAENV in DOCKERMERGED.containers['so-elastic-agent'].extra_env %}
- {{ XTRAENV }}
{% endfor %}
{% endif %}
{% if DOCKERMERGED.containers['so-elastic-agent'].ulimits %}
- ulimits:
{% for ULIMIT in DOCKERMERGED.containers['so-elastic-agent'].ulimits %}
- {{ ULIMIT.name }}={{ ULIMIT.soft }}:{{ ULIMIT.hard }}
{% endfor %}
{% endif %}
- require:
- file: create-elastic-agent-config
- file: trusttheca
- watch:
- file: create-elastic-agent-config
- file: trusttheca
delete_so-elastic-agent_so-status.disabled:
file.uncomment:

411
salt/elasticagent/files/elastic-agent.yml.jinja
View File

@@ -3,7 +3,7 @@
{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %}
id: aea1ba80-1065-11ee-a369-97538913b6a9
revision: 1
revision: 4
outputs:
default:
type: elasticsearch
@@ -22,242 +22,133 @@ agent:
metrics: false
features: {}
inputs:
- id: logfile-logs-fefef78c-422f-4cfa-8abf-4cd1b9428f62
name: import-evtx-logs
revision: 2
type: logfile
use_output: default
meta:
package:
name: log
version:
data_stream:
namespace: so
package_policy_id: fefef78c-422f-4cfa-8abf-4cd1b9428f62
streams:
- id: logfile-log.log-fefef78c-422f-4cfa-8abf-4cd1b9428f62
data_stream:
dataset: import
paths:
- /nsm/import/*/evtx/*.json
processors:
- dissect:
field: log.file.path
tokenizer: '/nsm/import/%{import.id}/evtx/%{import.file}'
target_prefix: ''
- decode_json_fields:
fields:
- message
target: ''
- drop_fields:
ignore_missing: true
fields:
- host
- add_fields:
fields:
dataset: system.security
type: logs
namespace: default
target: data_stream
- add_fields:
fields:
dataset: system.security
module: system
imported: true
target: event
- then:
- add_fields:
fields:
dataset: windows.sysmon_operational
target: data_stream
- add_fields:
fields:
dataset: windows.sysmon_operational
module: windows
imported: true
target: event
if:
equals:
winlog.channel: Microsoft-Windows-Sysmon/Operational
- then:
- add_fields:
fields:
dataset: system.application
target: data_stream
- add_fields:
fields:
dataset: system.application
target: event
if:
equals:
winlog.channel: Application
- then:
- add_fields:
fields:
dataset: system.system
target: data_stream
- add_fields:
fields:
dataset: system.system
target: event
if:
equals:
winlog.channel: System
- then:
- add_fields:
fields:
dataset: windows.powershell_operational
target: data_stream
- add_fields:
fields:
dataset: windows.powershell_operational
module: windows
target: event
if:
equals:
winlog.channel: Microsoft-Windows-PowerShell/Operational
tags:
- import
- id: logfile-redis-fc98c947-7d17-4861-a318-7ad075f6d1b0
name: redis-logs
revision: 2
type: logfile
use_output: default
meta:
package:
name: redis
version:
data_stream:
namespace: default
package_policy_id: fc98c947-7d17-4861-a318-7ad075f6d1b0
streams:
- id: logfile-redis.log-fc98c947-7d17-4861-a318-7ad075f6d1b0
data_stream:
dataset: redis.log
type: logs
exclude_files:
- .gz$
paths:
- /opt/so/log/redis/redis.log
tags:
- redis-log
exclude_lines:
- '^\s+[\-`(''.|_]'
- id: logfile-logs-3b56803d-5ade-4c93-b25e-9b37182f66b8
- id: filestream-filestream-85820eb0-25ef-11f0-a18d-1b26f69b8310
name: import-suricata-logs
revision: 2
type: logfile
revision: 3
type: filestream
use_output: default
meta:
package:
name: log
name: filestream
version:
data_stream:
namespace: so
package_policy_id: 3b56803d-5ade-4c93-b25e-9b37182f66b8
package_policy_id: 85820eb0-25ef-11f0-a18d-1b26f69b8310
streams:
- id: logfile-log.log-3b56803d-5ade-4c93-b25e-9b37182f66b8
- id: filestream-filestream.generic-85820eb0-25ef-11f0-a18d-1b26f69b8310
data_stream:
dataset: import
pipeline: suricata.common
paths:
- /nsm/import/*/suricata/eve*.json
pipeline: suricata.common
prospector.scanner.recursive_glob: true
prospector.scanner.exclude_files:
- \.gz$
ignore_older: 72h
clean_inactive: -1
parsers: null
processors:
- add_fields:
target: event
fields:
category: network
module: suricata
imported: true
category: network
target: event
- dissect:
tokenizer: /nsm/import/%{import.id}/suricata/%{import.file}
field: log.file.path
tokenizer: '/nsm/import/%{import.id}/suricata/%{import.file}'
target_prefix: ''
- id: logfile-logs-c327e1a3-1ebe-449c-a8eb-f6f35032e69d
name: soc-server-logs
revision: 2
type: logfile
file_identity.native: null
prospector.scanner.fingerprint.enabled: false
- id: filestream-filestream-86b4e960-25ef-11f0-a18d-1b26f69b8310
name: import-zeek-logs
revision: 3
type: filestream
use_output: default
meta:
package:
name: log
name: filestream
version:
data_stream:
namespace: so
package_policy_id: c327e1a3-1ebe-449c-a8eb-f6f35032e69d
package_policy_id: 86b4e960-25ef-11f0-a18d-1b26f69b8310
streams:
- id: logfile-log.log-c327e1a3-1ebe-449c-a8eb-f6f35032e69d
- id: filestream-filestream.generic-86b4e960-25ef-11f0-a18d-1b26f69b8310
data_stream:
dataset: soc
pipeline: common
dataset: import
paths:
- /opt/so/log/soc/sensoroni-server.log
- /nsm/import/*/zeek/logs/*.log
prospector.scanner.recursive_glob: true
prospector.scanner.exclude_files:
- >-
(broker|capture_loss|cluster|conn-summary|console|ecat_arp_info|known_certs|known_hosts|known_services|loaded_scripts|ntp|ocsp|packet_filter|reporter|stats|stderr|stdout).log$
clean_inactive: -1
parsers: null
processors:
- decode_json_fields:
add_error_key: true
process_array: true
max_depth: 2
fields:
- message
target: soc
- dissect:
tokenizer: /nsm/import/%{import.id}/zeek/logs/%{import.file}
field: log.file.path
target_prefix: ''
- script:
lang: javascript
source: |
function process(event) {
var pl = event.Get("import.file").slice(0,-4);
event.Put("@metadata.pipeline", "zeek." + pl);
}
- add_fields:
fields:
module: soc
dataset_temp: server
category: host
target: event
- rename:
ignore_missing: true
fields:
- from: soc.fields.sourceIp
to: source.ip
- from: soc.fields.status
to: http.response.status_code
- from: soc.fields.method
to: http.request.method
- from: soc.fields.path
to: url.path
- from: soc.message
to: event.action
- from: soc.level
to: log.level
tags:
- so-soc
- id: logfile-logs-906e0d4c-9ec3-4c6a-bef6-e347ec9fd073
category: network
module: zeek
imported: true
- add_tags:
tags: ics
when:
regexp:
import.file: >-
^bacnet*|^bsap*|^cip*|^cotp*|^dnp3*|^ecat*|^enip*|^modbus*|^opcua*|^profinet*|^s7comm*
file_identity.native: null
prospector.scanner.fingerprint.enabled: false
- id: filestream-filestream-91741240-25ef-11f0-a18d-1b26f69b8310
name: soc-sensoroni-logs
revision: 2
type: logfile
revision: 3
type: filestream
use_output: default
meta:
package:
name: log
name: filestream
version:
data_stream:
namespace: so
package_policy_id: 906e0d4c-9ec3-4c6a-bef6-e347ec9fd073
package_policy_id: 91741240-25ef-11f0-a18d-1b26f69b8310
streams:
- id: logfile-log.log-906e0d4c-9ec3-4c6a-bef6-e347ec9fd073
- id: filestream-filestream.generic-91741240-25ef-11f0-a18d-1b26f69b8310
data_stream:
dataset: soc
pipeline: common
paths:
- /opt/so/log/sensoroni/sensoroni.log
pipeline: common
prospector.scanner.recursive_glob: true
prospector.scanner.exclude_files:
- \.gz$
clean_inactive: -1
parsers: null
processors:
- decode_json_fields:
add_error_key: true
process_array: true
max_depth: 2
fields:
- message
target: sensoroni
process_array: true
max_depth: 2
add_error_key: true
- add_fields:
target: event
fields:
category: host
module: soc
dataset_temp: sensoroni
category: host
target: event
- rename:
ignore_missing: true
fields:
- from: sensoroni.fields.sourceIp
to: source.ip
@@ -271,141 +162,100 @@ inputs:
to: event.action
- from: sensoroni.level
to: log.level
- id: logfile-logs-df0d7f2c-221f-433b-b18b-d1cf83250515
name: soc-salt-relay-logs
revision: 2
type: logfile
use_output: default
meta:
package:
name: log
version:
data_stream:
namespace: so
package_policy_id: df0d7f2c-221f-433b-b18b-d1cf83250515
streams:
- id: logfile-log.log-df0d7f2c-221f-433b-b18b-d1cf83250515
data_stream:
dataset: soc
pipeline: common
paths:
- /opt/so/log/soc/salt-relay.log
processors:
- dissect:
field: message
tokenizer: '%{soc.ts} | %{event.action}'
target_prefix: ''
- add_fields:
fields:
module: soc
dataset_temp: salt_relay
category: host
target: event
tags:
- so-soc
- id: logfile-logs-74bd2366-fe52-493c-bddc-843a017fc4d0
name: soc-auth-sync-logs
revision: 2
type: logfile
use_output: default
meta:
package:
name: log
version:
data_stream:
namespace: so
package_policy_id: 74bd2366-fe52-493c-bddc-843a017fc4d0
streams:
- id: logfile-log.log-74bd2366-fe52-493c-bddc-843a017fc4d0
data_stream:
dataset: soc
pipeline: common
paths:
- /opt/so/log/soc/sync.log
processors:
- dissect:
field: message
tokenizer: '%{event.action}'
target_prefix: ''
- add_fields:
fields:
module: soc
dataset_temp: auth_sync
category: host
target: event
tags:
- so-soc
- id: logfile-logs-d151d9bf-ff2a-4529-9520-c99244bc0253
ignore_missing: true
file_identity.native: null
prospector.scanner.fingerprint.enabled: false
- id: filestream-filestream-976e3900-25ef-11f0-a18d-1b26f69b8310
name: suricata-logs
revision: 2
type: logfile
revision: 3
type: filestream
use_output: default
meta:
package:
name: log
name: filestream
version:
data_stream:
namespace: so
package_policy_id: d151d9bf-ff2a-4529-9520-c99244bc0253
package_policy_id: 976e3900-25ef-11f0-a18d-1b26f69b8310
streams:
- id: logfile-log.log-d151d9bf-ff2a-4529-9520-c99244bc0253
- id: filestream-filestream.generic-976e3900-25ef-11f0-a18d-1b26f69b8310
data_stream:
dataset: suricata
pipeline: suricata.common
paths:
- /nsm/suricata/eve*.json
pipeline: suricata.common
prospector.scanner.recursive_glob: true
prospector.scanner.exclude_files:
- \.gz$
clean_inactive: -1
parsers: null
processors:
- add_fields:
fields:
module: suricata
category: network
target: event
- id: logfile-logs-31f94d05-ae75-40ee-b9c5-0e0356eff327
fields:
category: network
module: suricata
file_identity.native: null
prospector.scanner.fingerprint.enabled: false
- id: filestream-filestream-95091fe0-25ef-11f0-a18d-1b26f69b8310
name: strelka-logs
revision: 2
type: logfile
revision: 3
type: filestream
use_output: default
meta:
package:
name: log
name: filestream
version:
data_stream:
namespace: so
package_policy_id: 31f94d05-ae75-40ee-b9c5-0e0356eff327
package_policy_id: 95091fe0-25ef-11f0-a18d-1b26f69b8310
streams:
- id: logfile-log.log-31f94d05-ae75-40ee-b9c5-0e0356eff327
- id: filestream-filestream.generic-95091fe0-25ef-11f0-a18d-1b26f69b8310
data_stream:
dataset: strelka
pipeline: strelka.file
paths:
- /nsm/strelka/log/strelka.log
pipeline: strelka.file
prospector.scanner.recursive_glob: true
prospector.scanner.exclude_files:
- \.gz$
clean_inactive: -1
parsers: null
processors:
- add_fields:
fields:
module: strelka
category: file
target: event
- id: logfile-logs-6197fe84-9b58-4d9b-8464-3d517f28808d
fields:
category: file
module: strelka
file_identity.native: null
prospector.scanner.fingerprint.enabled: false
- id: filestream-filestream-9f309ca0-25ef-11f0-a18d-1b26f69b8310
name: zeek-logs
revision: 1
type: logfile
revision: 2
type: filestream
use_output: default
meta:
package:
name: log
version:
name: filestream
version:
data_stream:
namespace: so
package_policy_id: 6197fe84-9b58-4d9b-8464-3d517f28808d
package_policy_id: 9f309ca0-25ef-11f0-a18d-1b26f69b8310
streams:
- id: logfile-log.log-6197fe84-9b58-4d9b-8464-3d517f28808d
- id: filestream-filestream.generic-9f309ca0-25ef-11f0-a18d-1b26f69b8310
data_stream:
dataset: zeek
paths:
- /nsm/zeek/logs/current/*.log
prospector.scanner.recursive_glob: true
prospector.scanner.exclude_files:
- >-
(broker|capture_loss|cluster|conn-summary|console|ecat_arp_info|known_certs|known_hosts|known_services|loaded_scripts|ntp|ocsp|packet_filter|reporter|stats|stderr|stdout).log$
clean_inactive: -1
parsers: null
processors:
- dissect:
tokenizer: '/nsm/zeek/logs/current/%{pipeline}.log'
tokenizer: /nsm/zeek/logs/current/%{pipeline}.log
field: log.file.path
trim_chars: .log
target_prefix: ''
@@ -427,18 +277,17 @@ inputs:
regexp:
pipeline: >-
^bacnet*|^bsap*|^cip*|^cotp*|^dnp3*|^ecat*|^enip*|^modbus*|^opcua*|^profinet*|^s7comm*
exclude_files:
- >-
broker|capture_loss|cluster|ecat_arp_info|known_hosts|known_services|loaded_scripts|ntp|ocsp|packet_filter|reporter|stats|stderr|stdout.log$
file_identity.native: null
prospector.scanner.fingerprint.enabled: false
- id: udp-udp-35051de0-46a5-11ee-8d5d-9f98c8182f60
name: syslog-udp-514
revision: 3
revision: 4
type: udp
use_output: default
meta:
package:
name: udp
version: 1.10.0
version:
data_stream:
namespace: so
package_policy_id: 35051de0-46a5-11ee-8d5d-9f98c8182f60
@@ -458,13 +307,13 @@ inputs:
- syslog
- id: tcp-tcp-33d37bb0-46a5-11ee-8d5d-9f98c8182f60
name: syslog-tcp-514
revision: 3
revision: 4
type: tcp
use_output: default
meta:
package:
name: tcp
version: 1.10.0
version:
data_stream:
namespace: so
package_policy_id: 33d37bb0-46a5-11ee-8d5d-9f98c8182f60

1
salt/elasticagent/soc_elasticagent.yaml
View File

@@ -1,4 +1,5 @@
elasticagent:
enabled:
description: Enables or disables the Elastic Agent process. This process must remain enabled to allow collection of node events.
forcedType: bool
advanced: True

4
salt/elasticfleet/config.sls
View File

@@ -11,6 +11,7 @@
include:
- elasticfleet.artifact_registry
- elasticfleet.ssl
# Add EA Group
elasticfleetgroup:
@@ -95,6 +96,9 @@ soresourcesrepoclone:
- rev: 'main'
- depth: 1
- force_reset: True
- retry:
attempts: 3
interval: 10
{% endif %}
elasticdefendconfdir:

32
salt/elasticfleet/enabled.sls
View File

@@ -6,16 +6,17 @@
{% from 'allowed_states.map.jinja' import allowed_states %}
{% if sls.split('.')[0] in allowed_states %}
{% from 'vars/globals.map.jinja' import GLOBALS %}
{% from 'docker/docker.map.jinja' import DOCKER %}
{% from 'docker/docker.map.jinja' import DOCKERMERGED %}
{% from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %}
{# This value is generated during node install and stored in minion pillar #}
{% set SERVICETOKEN = salt['pillar.get']('elasticfleet:config:server:es_token','') %}
include:
- ca
- logstash.ssl
- elasticfleet.config
- elasticfleet.sostatus
- ssl
{% if grains.role not in ['so-fleet'] %}
# Wait for Elasticsearch to be ready - no reason to try running Elastic Fleet server if ES is not ready
@@ -93,17 +94,17 @@ so-elastic-fleet:
- user: 947
- networks:
- sobridge:
- ipv4_address: {{ DOCKER.containers['so-elastic-fleet'].ip }}
- ipv4_address: {{ DOCKERMERGED.containers['so-elastic-fleet'].ip }}
- extra_hosts:
- {{ GLOBALS.manager }}:{{ GLOBALS.manager_ip }}
- {{ GLOBALS.hostname }}:{{ GLOBALS.node_ip }}
{% if DOCKER.containers['so-elastic-fleet'].extra_hosts %}
{% for XTRAHOST in DOCKER.containers['so-elastic-fleet'].extra_hosts %}
{% if DOCKERMERGED.containers['so-elastic-fleet'].extra_hosts %}
{% for XTRAHOST in DOCKERMERGED.containers['so-elastic-fleet'].extra_hosts %}
- {{ XTRAHOST }}
{% endfor %}
{% endif %}
- port_bindings:
{% for BINDING in DOCKER.containers['so-elastic-fleet'].port_bindings %}
{% for BINDING in DOCKERMERGED.containers['so-elastic-fleet'].port_bindings %}
- {{ BINDING }}
{% endfor %}
- binds:
@@ -111,8 +112,8 @@ so-elastic-fleet:
- /etc/pki/elasticfleet-server.key:/etc/pki/elasticfleet-server.key:ro
- /etc/pki/tls/certs/intca.crt:/etc/pki/tls/certs/intca.crt:ro
- /opt/so/log/elasticfleet:/usr/share/elastic-agent/logs
{% if DOCKER.containers['so-elastic-fleet'].custom_bind_mounts %}
{% for BIND in DOCKER.containers['so-elastic-fleet'].custom_bind_mounts %}
{% if DOCKERMERGED.containers['so-elastic-fleet'].custom_bind_mounts %}
{% for BIND in DOCKERMERGED.containers['so-elastic-fleet'].custom_bind_mounts %}
- {{ BIND }}
{% endfor %}
{% endif %}
@@ -127,12 +128,23 @@ so-elastic-fleet:
- FLEET_CA=/etc/pki/tls/certs/intca.crt
- FLEET_SERVER_ELASTICSEARCH_CA=/etc/pki/tls/certs/intca.crt
- LOGS_PATH=logs
{% if DOCKER.containers['so-elastic-fleet'].extra_env %}
{% for XTRAENV in DOCKER.containers['so-elastic-fleet'].extra_env %}
{% if DOCKERMERGED.containers['so-elastic-fleet'].extra_env %}
{% for XTRAENV in DOCKERMERGED.containers['so-elastic-fleet'].extra_env %}
- {{ XTRAENV }}
{% endfor %}
{% endif %}
{% if DOCKERMERGED.containers['so-elastic-fleet'].ulimits %}
- ulimits:
{% for ULIMIT in DOCKERMERGED.containers['so-elastic-fleet'].ulimits %}
- {{ ULIMIT.name }}={{ ULIMIT.soft }}:{{ ULIMIT.hard }}
{% endfor %}
{% endif %}
- watch:
- file: trusttheca
- x509: etc_elasticfleet_key
- x509: etc_elasticfleet_crt
- require:
- file: trusttheca
- x509: etc_elasticfleet_key
- x509: etc_elasticfleet_crt
{% endif %}

0
salt/suricata/rules/PLACEHOLDER → salt/elasticfleet/files/certs/.gitkeep
View File

22
salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/import-zeek-logs.json
View File

@@ -2,7 +2,7 @@
{%- raw -%}
{
"package": {
"name": "log",
"name": "filestream",
"version": ""
},
"name": "import-zeek-logs",
@@ -10,19 +10,31 @@
"description": "Zeek Import logs",
"policy_id": "so-grid-nodes_general",
"inputs": {
"logs-logfile": {
"filestream-filestream": {
"enabled": true,
"streams": {
"log.logs": {
"filestream.generic": {
"enabled": true,
"vars": {
"paths": [
"/nsm/import/*/zeek/logs/*.log"
],
"data_stream.dataset": "import",
"tags": [],
"pipeline": "",
"parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n",
"exclude_files": ["({%- endraw -%}{{ ELASTICFLEETMERGED.logging.zeek.excluded | join('|') }}{%- raw -%})(\\..+)?\\.log$"],
"include_files": [],
"processors": "- dissect:\n tokenizer: \"/nsm/import/%{import.id}/zeek/logs/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"\n- script:\n lang: javascript\n source: >\n function process(event) {\n var pl = event.Get(\"import.file\").slice(0,-4);\n event.Put(\"@metadata.pipeline\", \"zeek.\" + pl);\n }\n- add_fields:\n target: event\n fields:\n category: network\n module: zeek\n imported: true\n- add_tags:\n tags: \"ics\"\n when:\n regexp:\n import.file: \"^bacnet*|^bsap*|^cip*|^cotp*|^dnp3*|^ecat*|^enip*|^modbus*|^opcua*|^profinet*|^s7comm*\"",
"custom": "exclude_files: [\"{%- endraw -%}{{ ELASTICFLEETMERGED.logging.zeek.excluded | join('|') }}{%- raw -%}.log$\"]\n"
"tags": [],
"recursive_glob": true,
"clean_inactive": -1,
"harvester_limit": 0,
"fingerprint": false,
"fingerprint_offset": 0,
"fingerprint_length": "64",
"file_identity_native": true,
"exclude_lines": [],
"include_lines": []
}
}
}

31
salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/kratos-logs.json
View File

@@ -11,36 +11,51 @@
{%- endif -%}
{
"package": {
"name": "log",
"name": "filestream",
"version": ""
},
"name": "kratos-logs",
"namespace": "so",
"description": "Kratos logs",
"policy_id": "so-grid-nodes_general",
"namespace": "so",
"inputs": {
"logs-logfile": {
"filestream-filestream": {
"enabled": true,
"streams": {
"log.logs": {
"filestream.generic": {
"enabled": true,
"vars": {
"paths": [
"/opt/so/log/kratos/kratos.log"
],
"data_stream.dataset": "kratos",
"tags": ["so-kratos"],
"pipeline": "kratos",
"parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n",
"exclude_files": [
"\\.gz$"
],
"include_files": [],
{%- if valid_identities -%}
"processors": "- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n add_error_key: true\n- add_fields:\n target: event\n fields:\n category: iam\n module: kratos\n- if:\n has_fields:\n - identity_id\n then:{% for id, email in identities %}\n - if:\n equals:\n identity_id: \"{{ id }}\"\n then:\n - add_fields:\n target: ''\n fields:\n user.name: \"{{ email }}\"{% endfor %}",
{%- else -%}
"processors": "- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n add_error_key: true\n- add_fields:\n target: event\n fields:\n category: iam\n module: kratos",
{%- endif -%}
"custom": "pipeline: kratos"
"tags": [
"so-kratos"
],
"recursive_glob": true,
"clean_inactive": -1,
"harvester_limit": 0,
"fingerprint": false,
"fingerprint_offset": 0,
"fingerprint_length": "64",
"file_identity_native": true,
"exclude_lines": [],
"include_lines": []
}
}
}
}
},
"force": true
}
}

24
salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/zeek-logs.json
View File

@@ -2,28 +2,38 @@
{%- raw -%}
{
"package": {
"name": "log",
"name": "filestream",
"version": ""
},
"id": "zeek-logs",
"name": "zeek-logs",
"namespace": "so",
"description": "Zeek logs",
"policy_id": "so-grid-nodes_general",
"inputs": {
"logs-logfile": {
"filestream-filestream": {
"enabled": true,
"streams": {
"log.logs": {
"filestream.generic": {
"enabled": true,
"vars": {
"paths": [
"/nsm/zeek/logs/current/*.log"
],
"data_stream.dataset": "zeek",
"tags": [],
"parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n",
"exclude_files": ["({%- endraw -%}{{ ELASTICFLEETMERGED.logging.zeek.excluded | join('|') }}{%- raw -%})(\\..+)?\\.log$"],
"include_files": [],
"processors": "- dissect:\n tokenizer: \"/nsm/zeek/logs/current/%{pipeline}.log\"\n field: \"log.file.path\"\n trim_chars: \".log\"\n target_prefix: \"\"\n- script:\n lang: javascript\n source: >\n function process(event) {\n var pl = event.Get(\"pipeline\");\n event.Put(\"@metadata.pipeline\", \"zeek.\" + pl);\n }\n- add_fields:\n target: event\n fields:\n category: network\n module: zeek\n- add_tags:\n tags: \"ics\"\n when:\n regexp:\n pipeline: \"^bacnet*|^bsap*|^cip*|^cotp*|^dnp3*|^ecat*|^enip*|^modbus*|^opcua*|^profinet*|^s7comm*\"",
"custom": "exclude_files: [\"{%- endraw -%}{{ ELASTICFLEETMERGED.logging.zeek.excluded | join('|') }}{%- raw -%}.log$\"]\n"
"tags": [],
"recursive_glob": true,
"clean_inactive": -1,
"harvester_limit": 0,
"fingerprint": false,
"fingerprint_offset": 0,
"fingerprint_length": "64",
"file_identity_native": true,
"exclude_lines": [],
"include_lines": []
}
}
}
@@ -31,4 +41,4 @@
},
"force": true
}
{%- endraw -%}
{%- endraw -%}

2
salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json
View File

@@ -5,7 +5,7 @@
"package": {
"name": "endpoint",
"title": "Elastic Defend",
"version": "8.18.1",
"version": "9.0.2",
"requires_root": true
},
"enabled": true,

33
salt/elasticfleet/files/integrations/grid-nodes_general/hydra-logs.json
View File

@@ -1,26 +1,43 @@
{
"package": {
"name": "log",
"name": "filestream",
"version": ""
},
"name": "hydra-logs",
"namespace": "so",
"description": "Hydra logs",
"policy_id": "so-grid-nodes_general",
"namespace": "so",
"inputs": {
"logs-logfile": {
"filestream-filestream": {
"enabled": true,
"streams": {
"log.logs": {
"filestream.generic": {
"enabled": true,
"vars": {
"paths": [
"/opt/so/log/hydra/hydra.log"
],
"data_stream.dataset": "hydra",
"tags": ["so-hydra"],
"processors": "- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n add_error_key: true \n- add_fields:\n target: event\n fields:\n category: iam\n module: hydra",
"custom": "pipeline: hydra"
"pipeline": "hydra",
"parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n",
"exclude_files": [
"\\.gz$"
],
"include_files": [],
"processors": "- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n add_error_key: true\n- add_fields:\n target: event\n fields:\n category: iam\n module: hydra",
"tags": [
"so-hydra"
],
"recursive_glob": true,
"ignore_older": "72h",
"clean_inactive": -1,
"harvester_limit": 0,
"fingerprint": false,
"fingerprint_offset": 0,
"fingerprint_length": "64",
"file_identity_native": true,
"exclude_lines": [],
"include_lines": []
}
}
}
@@ -28,3 +45,5 @@
},
"force": true
}

28
salt/elasticfleet/files/integrations/grid-nodes_general/idh-logs.json
View File

@@ -1,30 +1,44 @@
{
"package": {
"name": "log",
"name": "filestream",
"version": ""
},
"name": "idh-logs",
"namespace": "so",
"description": "IDH integration",
"policy_id": "so-grid-nodes_general",
"namespace": "so",
"inputs": {
"logs-logfile": {
"filestream-filestream": {
"enabled": true,
"streams": {
"log.logs": {
"filestream.generic": {
"enabled": true,
"vars": {
"paths": [
"/nsm/idh/opencanary.log"
],
"data_stream.dataset": "idh",
"tags": [],
"pipeline": "common",
"parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n",
"exclude_files": [
"\\.gz$"
],
"include_files": [],
"processors": "\n- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n add_error_key: true\n- convert:\n fields:\n - {from: \"logtype\", to: \"event.code\", type: \"string\"}\n- drop_fields:\n when:\n equals:\n event.code: \"1001\"\n fields: [\"src_host\", \"src_port\", \"dst_host\", \"dst_port\" ]\n ignore_missing: true\n- rename:\n fields:\n - from: \"src_host\"\n to: \"source.ip\"\n - from: \"src_port\"\n to: \"source.port\"\n - from: \"dst_host\"\n to: \"destination.host\"\n - from: \"dst_port\"\n to: \"destination.port\"\n ignore_missing: true\n- drop_fields:\n fields: '[\"prospector\", \"input\", \"offset\", \"beat\"]'\n- add_fields:\n target: event\n fields:\n category: host\n module: opencanary",
"custom": "pipeline: common"
"tags": [],
"recursive_glob": true,
"clean_inactive": -1,
"harvester_limit": 0,
"fingerprint": false,
"fingerprint_offset": 0,
"fingerprint_length": "64",
"file_identity_native": true,
"exclude_lines": [],
"include_lines": []
}
}
}
}
},
"force": true
}
}

29
salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json
View File

@@ -1,33 +1,46 @@
{
"package": {
"name": "log",
"name": "filestream",
"version": ""
},
"name": "import-evtx-logs",
"namespace": "so",
"description": "Import Windows EVTX logs",
"policy_id": "so-grid-nodes_general",
"vars": {},
"namespace": "so",
"inputs": {
"logs-logfile": {
"filestream-filestream": {
"enabled": true,
"streams": {
"log.logs": {
"filestream.generic": {
"enabled": true,
"vars": {
"paths": [
"/nsm/import/*/evtx/*.json"
],
"data_stream.dataset": "import",
"custom": "",
"parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n",
"exclude_files": [
"\\.gz$"
],
"include_files": [],
"processors": "- dissect:\n tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"\n- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n- drop_fields:\n fields: [\"host\"]\n ignore_missing: true\n- add_fields:\n target: data_stream\n fields:\n type: logs\n dataset: system.security\n- add_fields:\n target: event\n fields:\n dataset: system.security\n module: system\n imported: true\n- add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.security-2.6.1\n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.sysmon_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.sysmon_operational\n module: windows\n imported: true\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.sysmon_operational-3.1.2\n- if:\n equals:\n winlog.channel: 'Application'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.application\n - add_fields:\n target: event\n fields:\n dataset: system.application\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.application-2.6.1\n- if:\n equals:\n winlog.channel: 'System'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.system\n - add_fields:\n target: event\n fields:\n dataset: system.system\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.system-2.6.1\n \n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.powershell_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.powershell_operational\n module: windows\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.powershell_operational-3.1.2\n- add_fields:\n target: data_stream\n fields:\n dataset: import",
"tags": [
"import"
]
],
"recursive_glob": true,
"ignore_older": "72h",
"clean_inactive": -1,
"harvester_limit": 0,
"fingerprint": false,
"fingerprint_offset": 0,
"fingerprint_length": "64",
"file_identity_native": true,
"exclude_lines": [],
"include_lines": []
}
}
}
}
},
"force": true
}
}

29
salt/elasticfleet/files/integrations/grid-nodes_general/import-suricata-logs.json
View File

@@ -1,30 +1,45 @@
{
"package": {
"name": "log",
"name": "filestream",
"version": ""
},
"name": "import-suricata-logs",
"namespace": "so",
"description": "Import Suricata logs",
"policy_id": "so-grid-nodes_general",
"namespace": "so",
"inputs": {
"logs-logfile": {
"filestream-filestream": {
"enabled": true,
"streams": {
"log.logs": {
"filestream.generic": {
"enabled": true,
"vars": {
"paths": [
"/nsm/import/*/suricata/eve*.json"
],
"data_stream.dataset": "import",
"pipeline": "suricata.common",
"parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n",
"exclude_files": [
"\\.gz$"
],
"include_files": [],
"processors": "- add_fields:\n target: event\n fields:\n category: network\n module: suricata\n imported: true\n- dissect:\n tokenizer: \"/nsm/import/%{import.id}/suricata/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"\n",
"tags": [],
"processors": "- add_fields:\n target: event\n fields:\n category: network\n module: suricata\n imported: true\n- dissect:\n tokenizer: \"/nsm/import/%{import.id}/suricata/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"",
"custom": "pipeline: suricata.common"
"recursive_glob": true,
"ignore_older": "72h",
"clean_inactive": -1,
"harvester_limit": 0,
"fingerprint": false,
"fingerprint_offset": 0,
"fingerprint_length": "64",
"file_identity_native": true,
"exclude_lines": [],
"include_lines": []
}
}
}
}
},
"force": true
}
}

2
salt/elasticfleet/files/integrations/grid-nodes_general/redis-logs.json
View File

@@ -15,7 +15,7 @@
"enabled": true,
"vars": {
"paths": [
"/opt/so/log/redis/redis.log"
"/opt/so/log/redis/redis-server.log"
],
"tags": [
"redis-log"

32
salt/elasticfleet/files/integrations/grid-nodes_general/rita-logs.json
View File

@@ -1,18 +1,17 @@
{
"package": {
"name": "log",
"name": "filestream",
"version": ""
},
"name": "rita-logs",
"namespace": "so",
"description": "RITA Logs",
"policy_id": "so-grid-nodes_general",
"vars": {},
"namespace": "so",
"inputs": {
"logs-logfile": {
"filestream-filestream": {
"enabled": true,
"streams": {
"log.logs": {
"filestream.generic": {
"enabled": true,
"vars": {
"paths": [
@@ -20,15 +19,28 @@
"/nsm/rita/exploded-dns.csv",
"/nsm/rita/long-connections.csv"
],
"exclude_files": [],
"ignore_older": "72h",
"data_stream.dataset": "rita",
"tags": [],
"parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n",
"exclude_files": [
"\\.gz$"
],
"include_files": [],
"processors": "- dissect:\n tokenizer: \"/nsm/rita/%{pipeline}.csv\"\n field: \"log.file.path\"\n trim_chars: \".csv\"\n target_prefix: \"\"\n- script:\n lang: javascript\n source: >\n function process(event) {\n var pl = event.Get(\"pipeline\").split(\"-\");\n if (pl.length > 1) {\n pl = pl[1];\n }\n else {\n pl = pl[0];\n }\n event.Put(\"@metadata.pipeline\", \"rita.\" + pl);\n }\n- add_fields:\n target: event\n fields:\n category: network\n module: rita",
"custom": "exclude_lines: ['^Score', '^Source', '^Domain', '^No results']"
"tags": [],
"recursive_glob": true,
"ignore_older": "72h",
"clean_inactive": -1,
"harvester_limit": 0,
"fingerprint": false,
"fingerprint_offset": 0,
"fingerprint_length": "64",
"file_identity_native": true,
"exclude_lines": [],
"include_lines": []
}
}
}
}
}
},
"force": true
}

28
salt/elasticfleet/files/integrations/grid-nodes_general/so-ip-mappings.json
View File

@@ -1,29 +1,41 @@
{
"package": {
"name": "log",
"name": "filestream",
"version": ""
},
"name": "so-ip-mappings",
"namespace": "so",
"description": "IP Description mappings",
"policy_id": "so-grid-nodes_general",
"vars": {},
"namespace": "so",
"inputs": {
"logs-logfile": {
"filestream-filestream": {
"enabled": true,
"streams": {
"log.logs": {
"filestream.generic": {
"enabled": true,
"vars": {
"paths": [
"/nsm/custom-mappings/ip-descriptions.csv"
],
"data_stream.dataset": "hostnamemappings",
"parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n",
"exclude_files": [
"\\.gz$"
],
"include_files": [],
"processors": "- decode_csv_fields:\n fields:\n message: decoded.csv\n separator: \",\"\n ignore_missing: false\n overwrite_keys: true\n trim_leading_space: true\n fail_on_error: true\n\n- extract_array:\n field: decoded.csv\n mappings:\n so.ip_address: '0'\n so.description: '1'\n\n- script:\n lang: javascript\n source: >\n function process(event) {\n var ip = event.Get('so.ip_address');\n var validIpRegex = /^((25[0-5]|2[0-4]\\d|1\\d{2}|[1-9]?\\d)\\.){3}(25[0-5]|2[0-4]\\d|1\\d{2}|[1-9]?\\d)$/\n if (!validIpRegex.test(ip)) {\n event.Cancel();\n }\n }\n- fingerprint:\n fields: [\"so.ip_address\"]\n target_field: \"@metadata._id\"\n",
"tags": [
"so-ip-mappings"
],
"processors": "- decode_csv_fields:\n fields:\n message: decoded.csv\n separator: \",\"\n ignore_missing: false\n overwrite_keys: true\n trim_leading_space: true\n fail_on_error: true\n\n- extract_array:\n field: decoded.csv\n mappings:\n so.ip_address: '0'\n so.description: '1'\n\n- script:\n lang: javascript\n source: >\n function process(event) {\n var ip = event.Get('so.ip_address');\n var validIpRegex = /^((25[0-5]|2[0-4]\\d|1\\d{2}|[1-9]?\\d)\\.){3}(25[0-5]|2[0-4]\\d|1\\d{2}|[1-9]?\\d)$/\n if (!validIpRegex.test(ip)) {\n event.Cancel();\n }\n }\n- fingerprint:\n fields: [\"so.ip_address\"]\n target_field: \"@metadata._id\"\n",
"custom": ""
"recursive_glob": true,
"clean_inactive": -1,
"harvester_limit": 0,
"fingerprint": false,
"fingerprint_offset": 0,
"fingerprint_length": "64",
"file_identity_native": true,
"exclude_lines": [],
"include_lines": []
}
}
}
@@ -31,5 +43,3 @@
},
"force": true
}

28
salt/elasticfleet/files/integrations/grid-nodes_general/soc-auth-sync-logs.json
View File

@@ -1,30 +1,44 @@
{
"package": {
"name": "log",
"name": "filestream",
"version": ""
},
"name": "soc-auth-sync-logs",
"namespace": "so",
"description": "Security Onion - Elastic Auth Sync - Logs",
"policy_id": "so-grid-nodes_general",
"namespace": "so",
"inputs": {
"logs-logfile": {
"filestream-filestream": {
"enabled": true,
"streams": {
"log.logs": {
"filestream.generic": {
"enabled": true,
"vars": {
"paths": [
"/opt/so/log/soc/sync.log"
],
"data_stream.dataset": "soc",
"tags": ["so-soc"],
"pipeline": "common",
"parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n",
"exclude_files": [
"\\.gz$"
],
"include_files": [],
"processors": "- dissect:\n tokenizer: \"%{event.action}\"\n field: \"message\"\n target_prefix: \"\"\n- add_fields:\n target: event\n fields:\n category: host\n module: soc\n dataset_temp: auth_sync",
"custom": "pipeline: common"
"tags": [],
"recursive_glob": true,
"clean_inactive": -1,
"harvester_limit": 0,
"fingerprint": false,
"fingerprint_offset": 0,
"fingerprint_length": "64",
"file_identity_native": true,
"exclude_lines": [],
"include_lines": []
}
}
}
}
},
"force": true
}
}

31
salt/elasticfleet/files/integrations/grid-nodes_general/soc-detections-logs.json
View File

@@ -1,35 +1,48 @@
{
"policy_id": "so-grid-nodes_general",
"package": {
"name": "log",
"name": "filestream",
"version": ""
},
"name": "soc-detections-logs",
"description": "Security Onion Console - Detections Logs",
"policy_id": "so-grid-nodes_general",
"namespace": "so",
"inputs": {
"logs-logfile": {
"filestream-filestream": {
"enabled": true,
"streams": {
"log.logs": {
"filestream.generic": {
"enabled": true,
"vars": {
"paths": [
"/opt/so/log/soc/detections_runtime-status_sigma.log",
"/opt/so/log/soc/detections_runtime-status_yara.log"
],
"exclude_files": [],
"ignore_older": "72h",
"data_stream.dataset": "soc",
"pipeline": "common",
"parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n",
"exclude_files": [
"\\.gz$"
],
"include_files": [],
"processors": "- decode_json_fields:\n fields: [\"message\"]\n target: \"soc\"\n process_array: true\n max_depth: 2\n add_error_key: true \n- add_fields:\n target: event\n fields:\n category: host\n module: soc\n dataset_temp: detections\n- rename:\n fields:\n - from: \"soc.fields.sourceIp\"\n to: \"source.ip\"\n - from: \"soc.fields.status\"\n to: \"http.response.status_code\"\n - from: \"soc.fields.method\"\n to: \"http.request.method\"\n - from: \"soc.fields.path\"\n to: \"url.path\"\n - from: \"soc.message\"\n to: \"event.action\"\n - from: \"soc.level\"\n to: \"log.level\"\n ignore_missing: true",
"tags": [
"so-soc"
],
"processors": "- decode_json_fields:\n fields: [\"message\"]\n target: \"soc\"\n process_array: true\n max_depth: 2\n add_error_key: true \n- add_fields:\n target: event\n fields:\n category: host\n module: soc\n dataset_temp: detections\n- rename:\n fields:\n - from: \"soc.fields.sourceIp\"\n to: \"source.ip\"\n - from: \"soc.fields.status\"\n to: \"http.response.status_code\"\n - from: \"soc.fields.method\"\n to: \"http.request.method\"\n - from: \"soc.fields.path\"\n to: \"url.path\"\n - from: \"soc.message\"\n to: \"event.action\"\n - from: \"soc.level\"\n to: \"log.level\"\n ignore_missing: true",
"custom": "pipeline: common"
"recursive_glob": true,
"ignore_older": "72h",
"clean_inactive": -1,
"harvester_limit": 0,
"fingerprint": false,
"fingerprint_offset": 0,
"fingerprint_length": "64",
"file_identity_native": true,
"exclude_lines": [],
"include_lines": []
}
}
}
}
},
"force": true
}
}

30
salt/elasticfleet/files/integrations/grid-nodes_general/soc-salt-relay-logs.json
View File

@@ -1,30 +1,46 @@
{
"package": {
"name": "log",
"name": "filestream",
"version": ""
},
"name": "soc-salt-relay-logs",
"namespace": "so",
"description": "Security Onion - Salt Relay - Logs",
"policy_id": "so-grid-nodes_general",
"namespace": "so",
"inputs": {
"logs-logfile": {
"filestream-filestream": {
"enabled": true,
"streams": {
"log.logs": {
"filestream.generic": {
"enabled": true,
"vars": {
"paths": [
"/opt/so/log/soc/salt-relay.log"
],
"data_stream.dataset": "soc",
"tags": ["so-soc"],
"pipeline": "common",
"parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n",
"exclude_files": [
"\\.gz$"
],
"include_files": [],
"processors": "- dissect:\n tokenizer: \"%{soc.ts} | %{event.action}\"\n field: \"message\"\n target_prefix: \"\"\n- add_fields:\n target: event\n fields:\n category: host\n module: soc\n dataset_temp: salt_relay",
"custom": "pipeline: common"
"tags": [
"so-soc"
],
"recursive_glob": true,
"clean_inactive": -1,
"harvester_limit": 0,
"fingerprint": false,
"fingerprint_offset": 0,
"fingerprint_length": "64",
"file_identity_native": true,
"exclude_lines": [],
"include_lines": []
}
}
}
}
},
"force": true
}
}

30
salt/elasticfleet/files/integrations/grid-nodes_general/soc-sensoroni-logs.json
View File

@@ -1,30 +1,44 @@
{
"package": {
"name": "log",
"name": "filestream",
"version": ""
},
"name": "soc-sensoroni-logs",
"namespace": "so",
"description": "Security Onion - Sensoroni - Logs",
"policy_id": "so-grid-nodes_general",
"namespace": "so",
"inputs": {
"logs-logfile": {
"filestream-filestream": {
"enabled": true,
"streams": {
"log.logs": {
"filestream.generic": {
"enabled": true,
"vars": {
"paths": [
"/opt/so/log/sensoroni/sensoroni.log"
],
"data_stream.dataset": "soc",
"tags": [],
"pipeline": "common",
"parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n",
"exclude_files": [
"\\.gz$"
],
"include_files": [],
"processors": "- decode_json_fields:\n fields: [\"message\"]\n target: \"sensoroni\"\n process_array: true\n max_depth: 2\n add_error_key: true \n- add_fields:\n target: event\n fields:\n category: host\n module: soc\n dataset_temp: sensoroni\n- rename:\n fields:\n - from: \"sensoroni.fields.sourceIp\"\n to: \"source.ip\"\n - from: \"sensoroni.fields.status\"\n to: \"http.response.status_code\"\n - from: \"sensoroni.fields.method\"\n to: \"http.request.method\"\n - from: \"sensoroni.fields.path\"\n to: \"url.path\"\n - from: \"sensoroni.message\"\n to: \"event.action\"\n - from: \"sensoroni.level\"\n to: \"log.level\"\n ignore_missing: true",
"custom": "pipeline: common"
"tags": [],
"recursive_glob": true,
"clean_inactive": -1,
"harvester_limit": 0,
"fingerprint": false,
"fingerprint_offset": 0,
"fingerprint_length": "64",
"file_identity_native": true,
"exclude_lines": [],
"include_lines": []
}
}
}
}
},
"force": true
}
"force": true
}

30
salt/elasticfleet/files/integrations/grid-nodes_general/soc-server-logs.json
View File

@@ -1,30 +1,46 @@
{
"package": {
"name": "log",
"name": "filestream",
"version": ""
},
"name": "soc-server-logs",
"namespace": "so",
"description": "Security Onion Console Logs",
"policy_id": "so-grid-nodes_general",
"namespace": "so",
"inputs": {
"logs-logfile": {
"filestream-filestream": {
"enabled": true,
"streams": {
"log.logs": {
"filestream.generic": {
"enabled": true,
"vars": {
"paths": [
"/opt/so/log/soc/sensoroni-server.log"
],
"data_stream.dataset": "soc",
"tags": ["so-soc"],
"pipeline": "common",
"parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n",
"exclude_files": [
"\\.gz$"
],
"include_files": [],
"processors": "- decode_json_fields:\n fields: [\"message\"]\n target: \"soc\"\n process_array: true\n max_depth: 2\n add_error_key: true \n- add_fields:\n target: event\n fields:\n category: host\n module: soc\n dataset_temp: server\n- rename:\n fields:\n - from: \"soc.fields.sourceIp\"\n to: \"source.ip\"\n - from: \"soc.fields.status\"\n to: \"http.response.status_code\"\n - from: \"soc.fields.method\"\n to: \"http.request.method\"\n - from: \"soc.fields.path\"\n to: \"url.path\"\n - from: \"soc.message\"\n to: \"event.action\"\n - from: \"soc.level\"\n to: \"log.level\"\n ignore_missing: true",
"custom": "pipeline: common"
"tags": [
"so-soc"
],
"recursive_glob": true,
"clean_inactive": -1,
"harvester_limit": 0,
"fingerprint": false,
"fingerprint_offset": 0,
"fingerprint_length": "64",
"file_identity_native": true,
"exclude_lines": [],
"include_lines": []
}
}
}
}
},
"force": true
}
}

30
salt/elasticfleet/files/integrations/grid-nodes_general/strelka-logs.json
View File

@@ -1,30 +1,44 @@
{
"package": {
"name": "log",
"name": "filestream",
"version": ""
},
"name": "strelka-logs",
"namespace": "so",
"description": "Strelka logs",
"description": "Strelka Logs",
"policy_id": "so-grid-nodes_general",
"namespace": "so",
"inputs": {
"logs-logfile": {
"filestream-filestream": {
"enabled": true,
"streams": {
"log.logs": {
"filestream.generic": {
"enabled": true,
"vars": {
"paths": [
"/nsm/strelka/log/strelka.log"
],
"data_stream.dataset": "strelka",
"tags": [],
"pipeline": "strelka.file",
"parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n",
"exclude_files": [
"\\.gz$"
],
"include_files": [],
"processors": "- add_fields:\n target: event\n fields:\n category: file\n module: strelka",
"custom": "pipeline: strelka.file"
"tags": [],
"recursive_glob": true,
"clean_inactive": -1,
"harvester_limit": 0,
"fingerprint": false,
"fingerprint_offset": 0,
"fingerprint_length": "64",
"file_identity_native": true,
"exclude_lines": [],
"include_lines": []
}
}
}
}
},
"force": true
}
}

26
salt/elasticfleet/files/integrations/grid-nodes_general/suricata-logs.json
View File

@@ -1,26 +1,40 @@
{
"package": {
"name": "log",
"name": "filestream",
"version": ""
},
"name": "suricata-logs",
"namespace": "so",
"description": "Suricata integration",
"policy_id": "so-grid-nodes_general",
"namespace": "so",
"inputs": {
"logs-logfile": {
"filestream-filestream": {
"enabled": true,
"streams": {
"log.logs": {
"filestream.generic": {
"enabled": true,
"vars": {
"paths": [
"/nsm/suricata/eve*.json"
],
"data_stream.dataset": "suricata",
"tags": [],
"pipeline": "suricata.common",
"parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n",
"exclude_files": [
"\\.gz$"
],
"include_files": [],
"processors": "- add_fields:\n target: event\n fields:\n category: network\n module: suricata",
"custom": "pipeline: suricata.common"
"tags": [],
"recursive_glob": true,
"clean_inactive": -1,
"harvester_limit": 0,
"fingerprint": false,
"fingerprint_offset": 0,
"fingerprint_length": "64",
"file_identity_native": true,
"exclude_lines": [],
"include_lines": []
}
}
}

107
salt/elasticfleet/files/integrations/grid-nodes_heavy/elasticsearch-grid-nodes.json Normal file
View File

@@ -0,0 +1,107 @@
{
"package": {
"name": "elasticsearch",
"version": ""
},
"name": "elasticsearch-grid-nodes_heavy",
"namespace": "default",
"description": "Elasticsearch Logs",
"policy_id": "so-grid-nodes_heavy",
"inputs": {
"elasticsearch-logfile": {
"enabled": true,
"streams": {
"elasticsearch.audit": {
"enabled": false,
"vars": {
"paths": [
"/var/log/elasticsearch/*_audit.json"
]
}
},
"elasticsearch.deprecation": {
"enabled": false,
"vars": {
"paths": [
"/var/log/elasticsearch/*_deprecation.json"
]
}
},
"elasticsearch.gc": {
"enabled": false,
"vars": {
"paths": [
"/var/log/elasticsearch/gc.log.[0-9]*",
"/var/log/elasticsearch/gc.log"
]
}
},
"elasticsearch.server": {
"enabled": true,
"vars": {
"paths": [
"/opt/so/log/elasticsearch/*.json"
]
}
},
"elasticsearch.slowlog": {
"enabled": false,
"vars": {
"paths": [
"/var/log/elasticsearch/*_index_search_slowlog.json",
"/var/log/elasticsearch/*_index_indexing_slowlog.json"
]
}
}
}
},
"elasticsearch-elasticsearch/metrics": {
"enabled": false,
"vars": {
"hosts": [
"http://localhost:9200"
],
"scope": "node"
},
"streams": {
"elasticsearch.stack_monitoring.ccr": {
"enabled": false
},
"elasticsearch.stack_monitoring.cluster_stats": {
"enabled": false
},
"elasticsearch.stack_monitoring.enrich": {
"enabled": false
},
"elasticsearch.stack_monitoring.index": {
"enabled": false
},
"elasticsearch.stack_monitoring.index_recovery": {
"enabled": false,
"vars": {
"active.only": true
}
},
"elasticsearch.stack_monitoring.index_summary": {
"enabled": false
},
"elasticsearch.stack_monitoring.ml_job": {
"enabled": false
},
"elasticsearch.stack_monitoring.node": {
"enabled": false
},
"elasticsearch.stack_monitoring.node_stats": {
"enabled": false
},
"elasticsearch.stack_monitoring.pending_tasks": {
"enabled": false
},
"elasticsearch.stack_monitoring.shard": {
"enabled": false
}
}
}
},
"force": true
}

6
salt/elasticfleet/install_agent_grid.sls
View File

@@ -8,7 +8,9 @@
{% endif %}
{% set AGENT_STATUS = salt['service.available']('elastic-agent') %}
{% if not AGENT_STATUS %}
{% set AGENT_EXISTS = salt['file.file_exists']('/opt/Elastic/Agent/elastic-agent') %}
{% if not AGENT_STATUS or not AGENT_EXISTS %}
pull_agent_installer:
file.managed:
@@ -19,7 +21,7 @@ pull_agent_installer:
run_installer:
cmd.run:
- name: ./so-elastic-agent_linux_amd64 -token={{ GRIDNODETOKEN }}
- name: ./so-elastic-agent_linux_amd64 -token={{ GRIDNODETOKEN }} -force
- cwd: /opt/so
- retry:
attempts: 3

1
salt/elasticfleet/integration-defaults.map.jinja
View File

@@ -21,6 +21,7 @@
'azure_application_insights.app_state': 'azure.app_state',
'azure_billing.billing': 'azure.billing',
'azure_functions.metrics': 'azure.function',
'azure_ai_foundry.metrics': 'azure.ai_foundry',
'azure_metrics.compute_vm_scaleset': 'azure.compute_vm_scaleset',
'azure_metrics.compute_vm': 'azure.compute_vm',
'azure_metrics.container_instance': 'azure.container_instance',

59
salt/elasticfleet/soc_elasticfleet.yaml
View File

@@ -1,14 +1,15 @@
elasticfleet:
enabled:
description: Enables or disables the Elastic Fleet process. This process is critical for managing Elastic Agents.
forcedType: bool
advanced: True
helpLink: elastic-fleet.html
helpLink: elastic-fleet
enable_manager_output:
description: Setting this option to False should only be considered if there is at least one receiver node in the grid. If True, Elastic Agent will send events to the manager and receivers. If False, events will only be send to the receivers.
advanced: True
global: True
forcedType: bool
helpLink: elastic-fleet.html
helpLink: elastic-fleet
files:
soc:
elastic-defend-disabled-filters__yaml:
@@ -17,7 +18,7 @@ elasticfleet:
syntax: yaml
file: True
global: True
helpLink: elastic-fleet.html
helpLink: elastic-fleet
advanced: True
elastic-defend-custom-filters__yaml:
title: Custom Elastic Defend filters
@@ -25,31 +26,32 @@ elasticfleet:
syntax: yaml
file: True
global: True
helpLink: elastic-fleet.html
helpLink: elastic-fleet
advanced: True
logging:
zeek:
excluded:
description: This is a list of Zeek logs that are excluded from being shipped through the data processing pipeline. If you remove a log from this list, Elastic Agent will attempt to process it. If an ingest node pipeline is not available to process the logs, you may experience errors.
forcedType: "[]string"
helpLink: zeek.html
helpLink: zeek
config:
defend_filters:
enable_auto_configuration:
description: Enable auto-configuration and management of the Elastic Defend Exclusion filters.
forcedType: bool
global: True
helpLink: elastic-fleet.html
helpLink: elastic-fleet
advanced: True
subscription_integrations:
description: Enable the installation of integrations that require an Elastic license.
global: True
forcedType: bool
helpLink: elastic-fleet.html
helpLink: elastic-fleet
auto_upgrade_integrations:
description: Enables or disables automatically upgrading Elastic Agent integrations.
global: True
forcedType: bool
helpLink: elastic-fleet.html
helpLink: elastic-fleet
outputs:
logstash:
bulk_max_size:
@@ -57,67 +59,68 @@ elasticfleet:
global: True
forcedType: int
advanced: True
helpLink: elastic-fleet.html
helpLink: elastic-fleet
worker:
description: The number of workers per configured host publishing events.
global: True
forcedType: int
advanced: true
helpLink: elastic-fleet.html
helpLink: elastic-fleet
queue_mem_events:
title: queued events
description: The number of events the queue can store. This value should be evenly divisible by the smaller of 'bulk_max_size' to avoid sending partial batches to the output.
global: True
forcedType: int
advanced: True
helpLink: elastic-fleet.html
helpLink: elastic-fleet
timeout:
description: The number of seconds to wait for responses from the Logstash server before timing out. Eg 30s
regex: ^[0-9]+s$
advanced: True
global: True
helpLink: elastic-fleet.html
helpLink: elastic-fleet
loadbalance:
description: If true and multiple Logstash hosts are configured, the output plugin load balances published events onto all Logstash hosts. If false, the output plugin sends all events to one host (determined at random) and switches to another host if the selected one becomes unresponsive.
forcedType: bool
advanced: True
global: True
helpLink: elastic-fleet.html
helpLink: elastic-fleet
compression_level:
description: The gzip compression level. The compression level must be in the range of 1 (best speed) to 9 (best compression).
regex: ^[1-9]$
forcedType: int
advanced: True
global: True
helpLink: elastic-fleet.html
helpLink: elastic-fleet
server:
custom_fqdn:
description: Custom FQDN for Agents to connect to. One per line.
global: True
helpLink: elastic-fleet.html
helpLink: elastic-fleet
advanced: True
forcedType: "[]string"
enable_auto_configuration:
description: Enable auto-configuration of Logstash Outputs & Fleet Host URLs.
forcedType: bool
global: True
helpLink: elastic-fleet.html
helpLink: elastic-fleet
advanced: True
endpoints_enrollment:
description: Endpoint enrollment key.
global: True
helpLink: elastic-fleet.html
helpLink: elastic-fleet
sensitive: True
advanced: True
es_token:
description: Elastic auth token.
global: True
helpLink: elastic-fleet.html
helpLink: elastic-fleet
sensitive: True
advanced: True
grid_enrollment:
description: Grid enrollment key.
global: True
helpLink: elastic-fleet.html
helpLink: elastic-fleet
sensitive: True
advanced: True
optional_integrations:
@@ -125,57 +128,57 @@ elasticfleet:
enabled_nodes:
description: Fleet nodes with the Sublime Platform integration enabled. Enter one per line.
global: True
helpLink: elastic-fleet.html
helpLink: elastic-fleet
advanced: True
forcedType: "[]string"
api_key:
description: API key for Sublime Platform.
global: True
helpLink: elastic-fleet.html
helpLink: elastic-fleet
advanced: True
forcedType: string
sensitive: True
base_url:
description: Base URL for Sublime Platform.
global: True
helpLink: elastic-fleet.html
helpLink: elastic-fleet
advanced: True
forcedType: string
poll_interval:
description: Poll interval for alerts from Sublime Platform.
global: True
helpLink: elastic-fleet.html
helpLink: elastic-fleet
advanced: True
forcedType: string
limit:
description: The maximum number of message groups to return from Sublime Platform.
global: True
helpLink: elastic-fleet.html
helpLink: elastic-fleet
advanced: True
forcedType: int
kismet:
base_url:
description: Base URL for Kismet.
global: True
helpLink: elastic-fleet.html
helpLink: elastic-fleet
advanced: True
forcedType: string
poll_interval:
description: Poll interval for wireless device data from Kismet. Integration is currently configured to return devices seen as active by any Kismet sensor within the last 10 minutes.
global: True
helpLink: elastic-fleet.html
helpLink: elastic-fleet
advanced: True
forcedType: string
api_key:
description: API key for Kismet.
global: True
helpLink: elastic-fleet.html
helpLink: elastic-fleet
advanced: True
forcedType: string
sensitive: True
enabled_nodes:
description: Fleet nodes with the Kismet integration enabled. Enter one per line.
global: True
helpLink: elastic-fleet.html
helpLink: elastic-fleet
advanced: True
forcedType: "[]string"

186
salt/elasticfleet/ssl.sls Normal file
View File

@@ -0,0 +1,186 @@
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{% from 'allowed_states.map.jinja' import allowed_states %}
{% if sls.split('.')[0] in allowed_states %}
{% from 'vars/globals.map.jinja' import GLOBALS %}
{% from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %}
{% from 'ca/map.jinja' import CA %}
{% if GLOBALS.is_manager or GLOBALS.role in ['so-heavynode', 'so-fleet', 'so-receiver'] %}
{% if grains['role'] not in [ 'so-heavynode', 'so-receiver'] %}
# Start -- Elastic Fleet Host Cert
etc_elasticfleet_key:
x509.private_key_managed:
- name: /etc/pki/elasticfleet-server.key
- keysize: 4096
- backup: True
- new: True
{% if salt['file.file_exists']('/etc/pki/elasticfleet-server.key') -%}
- prereq:
- x509: etc_elasticfleet_crt
{%- endif %}
- retry:
attempts: 5
interval: 30
etc_elasticfleet_crt:
x509.certificate_managed:
- name: /etc/pki/elasticfleet-server.crt
- ca_server: {{ CA.server }}
- signing_policy: elasticfleet
- private_key: /etc/pki/elasticfleet-server.key
- CN: {{ GLOBALS.hostname }}
- subjectAltName: DNS:{{ GLOBALS.hostname }},DNS:{{ GLOBALS.url_base }},IP:{{ GLOBALS.node_ip }}{% if ELASTICFLEETMERGED.config.server.custom_fqdn | length > 0 %},DNS:{{ ELASTICFLEETMERGED.config.server.custom_fqdn | join(',DNS:') }}{% endif %}
- days_remaining: 7
- days_valid: 820
- backup: True
- timeout: 30
- retry:
attempts: 5
interval: 30
efperms:
file.managed:
- replace: False
- name: /etc/pki/elasticfleet-server.key
- mode: 640
- group: 939
chownelasticfleetcrt:
file.managed:
- replace: False
- name: /etc/pki/elasticfleet-server.crt
- mode: 640
- user: 947
- group: 939
chownelasticfleetkey:
file.managed:
- replace: False
- name: /etc/pki/elasticfleet-server.key
- mode: 640
- user: 947
- group: 939
# End -- Elastic Fleet Host Cert
{% endif %} # endif is for not including HeavyNodes & Receivers
# Start -- Elastic Fleet Client Cert for Agent (Mutual Auth with Logstash Output)
etc_elasticfleet_agent_key:
x509.private_key_managed:
- name: /etc/pki/elasticfleet-agent.key
- keysize: 4096
- backup: True
- new: True
{% if salt['file.file_exists']('/etc/pki/elasticfleet-agent.key') -%}
- prereq:
- x509: etc_elasticfleet_agent_crt
{%- endif %}
- retry:
attempts: 5
interval: 30
etc_elasticfleet_agent_crt:
x509.certificate_managed:
- name: /etc/pki/elasticfleet-agent.crt
- ca_server: {{ CA.server }}
- signing_policy: elasticfleet
- private_key: /etc/pki/elasticfleet-agent.key
- CN: {{ GLOBALS.hostname }}
- days_remaining: 7
- days_valid: 820
- backup: True
- timeout: 30
- retry:
attempts: 5
interval: 30
cmd.run:
- name: "/usr/bin/openssl pkcs8 -in /etc/pki/elasticfleet-agent.key -topk8 -out /etc/pki/elasticfleet-agent.p8 -nocrypt"
- onchanges:
- x509: etc_elasticfleet_agent_key
efagentperms:
file.managed:
- replace: False
- name: /etc/pki/elasticfleet-agent.key
- mode: 640
- group: 939
chownelasticfleetagentcrt:
file.managed:
- replace: False
- name: /etc/pki/elasticfleet-agent.crt
- mode: 640
- user: 947
- group: 939
chownelasticfleetagentkey:
file.managed:
- replace: False
- name: /etc/pki/elasticfleet-agent.key
- mode: 640
- user: 947
- group: 939
# End -- Elastic Fleet Client Cert for Agent (Mutual Auth with Logstash Output)
{% endif %}
{% if GLOBALS.role in ['so-manager', 'so-managerhype', 'so-managersearch', 'so-standalone'] %}
elasticfleet_kafka_key:
x509.private_key_managed:
- name: /etc/pki/elasticfleet-kafka.key
- keysize: 4096
- backup: True
- new: True
{% if salt['file.file_exists']('/etc/pki/elasticfleet-kafka.key') -%}
- prereq:
- x509: elasticfleet_kafka_crt
{%- endif %}
- retry:
attempts: 5
interval: 30
elasticfleet_kafka_crt:
x509.certificate_managed:
- name: /etc/pki/elasticfleet-kafka.crt
- ca_server: {{ CA.server }}
- signing_policy: kafka
- private_key: /etc/pki/elasticfleet-kafka.key
- CN: {{ GLOBALS.hostname }}
- subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }}
- days_remaining: 7
- days_valid: 820
- backup: True
- timeout: 30
- retry:
attempts: 5
interval: 30
elasticfleet_kafka_cert_perms:
file.managed:
- replace: False
- name: /etc/pki/elasticfleet-kafka.crt
- mode: 640
- user: 947
- group: 939
elasticfleet_kafka_key_perms:
file.managed:
- replace: False
- name: /etc/pki/elasticfleet-kafka.key
- mode: 640
- user: 947
- group: 939
{% endif %}
{% else %}
{{sls}}_state_not_allowed:
test.fail_without_changes:
- name: {{sls}}_state_not_allowed
{% endif %}

31
salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load
View File

@@ -17,9 +17,9 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then
# Third, configure Elastic Defend Integration seperately
/usr/sbin/so-elastic-fleet-integration-policy-elastic-defend
# Initial Endpoints
for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/endpoints-initial/*.json
do
for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/endpoints-initial/*.json; do
printf "\n\nInitial Endpoints Policy - Loading $INTEGRATION\n"
elastic_fleet_integration_check "endpoints-initial" "$INTEGRATION"
if [ -n "$INTEGRATION_ID" ]; then
@@ -40,8 +40,7 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then
done
# Grid Nodes - General
for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/grid-nodes_general/*.json
do
for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/grid-nodes_general/*.json; do
printf "\n\nGrid Nodes Policy_General - Loading $INTEGRATION\n"
elastic_fleet_integration_check "so-grid-nodes_general" "$INTEGRATION"
if [ -n "$INTEGRATION_ID" ]; then
@@ -60,13 +59,9 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then
fi
fi
done
if [[ "$RETURN_CODE" != "1" ]]; then
touch /opt/so/state/eaintegrations.txt
fi
# Grid Nodes - Heavy
for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/grid-nodes_heavy/*.json
do
for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/grid-nodes_heavy/*.json; do
printf "\n\nGrid Nodes Policy_Heavy - Loading $INTEGRATION\n"
elastic_fleet_integration_check "so-grid-nodes_heavy" "$INTEGRATION"
if [ -n "$INTEGRATION_ID" ]; then
@@ -78,22 +73,16 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then
fi
else
printf "\n\nIntegration does not exist - Creating integration\n"
if [ "$NAME" != "elasticsearch-logs" ]; then
if ! elastic_fleet_integration_create "@$INTEGRATION"; then
echo -e "\nFailed to create integration for ${INTEGRATION##*/}"
RETURN_CODE=1
continue
fi
if ! elastic_fleet_integration_create "@$INTEGRATION"; then
echo -e "\nFailed to create integration for ${INTEGRATION##*/}"
RETURN_CODE=1
continue
fi
fi
done
if [[ "$RETURN_CODE" != "1" ]]; then
touch /opt/so/state/eaintegrations.txt
fi
# Fleet Server - Optional integrations
for INTEGRATION in /opt/so/conf/elastic-fleet/integrations-optional/FleetServer*/*.json
do
for INTEGRATION in /opt/so/conf/elastic-fleet/integrations-optional/FleetServer*/*.json; do
if ! [ "$INTEGRATION" == "/opt/so/conf/elastic-fleet/integrations-optional/FleetServer*/*.json" ]; then
FLEET_POLICY=`echo "$INTEGRATION"| cut -d'/' -f7`
printf "\n\nFleet Server Policy - Loading $INTEGRATION\n"
@@ -117,6 +106,8 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then
fi
fi
done
# Only create the state file if all policies were created/updated successfully
if [[ "$RETURN_CODE" != "1" ]]; then
touch /opt/so/state/eaintegrations.txt
fi

2
salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-grid-upgrade
View File

@@ -14,7 +14,7 @@ if ! is_manager_node; then
fi
# Get current list of Grid Node Agents that need to be upgraded
RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/agents?perPage=20&page=1&kuery=NOT%20agent.version%20:%20%22{{ELASTICSEARCHDEFAULTS.elasticsearch.version}}%22%20and%20policy_id%20:%20%22so-grid-nodes_general%22&showInactive=false&getStatusSummary=true")
RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/agents?perPage=20&page=1&kuery=NOT%20agent.version%3A%20{{ELASTICSEARCHDEFAULTS.elasticsearch.version}}%20AND%20policy_id%3A%20so-grid-nodes_%2A&showInactive=false&getStatusSummary=true" --retry 3 --retry-delay 30 --fail 2>/dev/null)
# Check to make sure that the server responded with good data - else, bail from script
CHECKSUM=$(jq -r '.page' <<< "$RAW_JSON")

2
salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-es-url-update
View File

@@ -26,7 +26,7 @@ function update_es_urls() {
}
# Get current list of Fleet Elasticsearch URLs
RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/outputs/so-manager_elasticsearch')
RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/outputs/so-manager_elasticsearch' --retry 3 --retry-delay 30 --fail 2>/dev/null)
# Check to make sure that the server responded with good data - else, bail from script
CHECKSUM=$(jq -r '.item.id' <<< "$RAW_JSON")

2
salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load
View File

@@ -86,7 +86,7 @@ if [[ -f $STATE_FILE_SUCCESS ]]; then
latest_package_list=$(/usr/sbin/so-elastic-fleet-package-list)
echo '{ "packages" : []}' > $BULK_INSTALL_PACKAGE_LIST
rm -f $INSTALLED_PACKAGE_LIST
echo $latest_package_list | jq '{packages: [.items[] | {name: .name, latest_version: .version, installed_version: .savedObject.attributes.install_version, subscription: .conditions.elastic.subscription }]}' >> $INSTALLED_PACKAGE_LIST
echo $latest_package_list | jq '{packages: [.items[] | {name: .name, latest_version: .version, installed_version: .installationInfo.version, subscription: .conditions.elastic.subscription }]}' >> $INSTALLED_PACKAGE_LIST
while read -r package; do
# get package details

4
salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update
View File

@@ -142,7 +142,7 @@ function update_kafka_outputs() {
{% if GLOBALS.pipeline == "KAFKA" %}
# Get current list of Kafka Outputs
RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/outputs/so-manager_kafka')
RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/outputs/so-manager_kafka' --retry 3 --retry-delay 30 --fail 2>/dev/null)
# Check to make sure that the server responded with good data - else, bail from script
CHECKSUM=$(jq -r '.item.id' <<< "$RAW_JSON")
@@ -168,7 +168,7 @@ function update_kafka_outputs() {
{# If global pipeline isn't set to KAFKA then assume default of REDIS / logstash #}
{% else %}
# Get current list of Logstash Outputs
RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/outputs/so-manager_logstash')
RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/outputs/so-manager_logstash' --retry 3 --retry-delay 30 --fail 2>/dev/null)
# Check to make sure that the server responded with good data - else, bail from script
CHECKSUM=$(jq -r '.item.id' <<< "$RAW_JSON")

2
salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup
View File

@@ -241,9 +241,11 @@ printf '%s\n'\
"" >> "$global_pillar_file"
# Call Elastic-Fleet Salt State
printf "\nApplying elasticfleet state"
salt-call state.apply elasticfleet queue=True
# Generate installers & install Elastic Agent on the node
so-elastic-agent-gen-installers
printf "\nApplying elasticfleet.install_agent_grid state"
salt-call state.apply elasticfleet.install_agent_grid queue=True
exit 0

2
salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update
View File

@@ -23,7 +23,7 @@ function update_fleet_urls() {
}
# Get current list of Fleet Server URLs
RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/fleet_server_hosts/grid-default')
RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/fleet_server_hosts/grid-default' --retry 3 --retry-delay 30 --fail 2>/dev/null)
# Check to make sure that the server responded with good data - else, bail from script
CHECKSUM=$(jq -r '.item.id' <<< "$RAW_JSON")

9
salt/elasticfleet/tools/sbin_jinja/so-kafka-fleet-output-policy
View File

@@ -34,6 +34,11 @@ if [[ "$RETURN_CODE" != "0" ]]; then
exit 1
fi
if [[ ! -f /etc/pki/elasticfleet-kafka.crt || ! -f /etc/pki/elasticfleet-kafka.key ]]; then
echo -e "\nKafka certificates not found, can't setup Elastic Fleet output policy for Kafka...\n"
exit 1
fi
KAFKACRT=$(openssl x509 -in /etc/pki/elasticfleet-kafka.crt)
KAFKAKEY=$(openssl rsa -in /etc/pki/elasticfleet-kafka.key)
KAFKACA=$(openssl x509 -in /etc/pki/tls/certs/intca.crt)
@@ -47,7 +52,7 @@ if ! kafka_output=$(curl -sK /opt/so/conf/elasticsearch/curl.config -L "http://l
--arg KAFKACA "$KAFKACA" \
--arg MANAGER_IP "{{ GLOBALS.manager_ip }}:9092" \
--arg KAFKA_OUTPUT_VERSION "$KAFKA_OUTPUT_VERSION" \
'{"name":"grid-kafka", "id":"so-manager_kafka","type":"kafka","hosts":[ $MANAGER_IP ],"is_default":false,"is_default_monitoring":false,"config_yaml":"","ssl":{"certificate_authorities":[ $KAFKACA ],"certificate": $KAFKACRT ,"key":"","verification_mode":"full"},"proxy_id":null,"client_id":"Elastic","version": $KAFKA_OUTPUT_VERSION ,"compression":"none","auth_type":"ssl","partition":"round_robin","round_robin":{"group_events":10},"topics":[{"topic":"default-securityonion"}],"headers":[{"key":"","value":""}],"timeout":30,"broker_timeout":30,"required_acks":1,"secrets":{"ssl":{"key": $KAFKAKEY }}}'
'{"name":"grid-kafka", "id":"so-manager_kafka","type":"kafka","hosts":[ $MANAGER_IP ],"is_default":false,"is_default_monitoring":false,"config_yaml":"","ssl":{"certificate_authorities":[ $KAFKACA ],"certificate": $KAFKACRT ,"key":"","verification_mode":"full"},"proxy_id":null,"client_id":"Elastic","version": $KAFKA_OUTPUT_VERSION ,"compression":"none","auth_type":"ssl","partition":"round_robin","round_robin":{"group_events":10},"topic":"default-securityonion","headers":[{"key":"","value":""}],"timeout":30,"broker_timeout":30,"required_acks":1,"secrets":{"ssl":{"key": $KAFKAKEY }}}'
)
if ! response=$(curl -sK /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" --fail 2>/dev/null); then
echo -e "\nFailed to setup Elastic Fleet output policy for Kafka...\n"
@@ -67,7 +72,7 @@ elif kafka_output=$(curl -sK /opt/so/conf/elasticsearch/curl.config -L "http://l
--arg ENABLED_DISABLED "$ENABLED_DISABLED"\
--arg KAFKA_OUTPUT_VERSION "$KAFKA_OUTPUT_VERSION" \
--argjson HOSTS "$HOSTS" \
'{"name":"grid-kafka","type":"kafka","hosts":$HOSTS,"is_default":$ENABLED_DISABLED,"is_default_monitoring":$ENABLED_DISABLED,"config_yaml":"","ssl":{"certificate_authorities":[ $KAFKACA ],"certificate": $KAFKACRT ,"key":"","verification_mode":"full"},"proxy_id":null,"client_id":"Elastic","version": $KAFKA_OUTPUT_VERSION ,"compression":"none","auth_type":"ssl","partition":"round_robin","round_robin":{"group_events":10},"topics":[{"topic":"default-securityonion"}],"headers":[{"key":"","value":""}],"timeout":30,"broker_timeout":30,"required_acks":1,"secrets":{"ssl":{"key": $KAFKAKEY }}}'
'{"name":"grid-kafka","type":"kafka","hosts":$HOSTS,"is_default":$ENABLED_DISABLED,"is_default_monitoring":$ENABLED_DISABLED,"config_yaml":"","ssl":{"certificate_authorities":[ $KAFKACA ],"certificate": $KAFKACRT ,"key":"","verification_mode":"full"},"proxy_id":null,"client_id":"Elastic","version": $KAFKA_OUTPUT_VERSION ,"compression":"none","auth_type":"ssl","partition":"round_robin","round_robin":{"group_events":10},"topic":"default-securityonion","headers":[{"key":"","value":""}],"timeout":30,"broker_timeout":30,"required_acks":1,"secrets":{"ssl":{"key": $KAFKAKEY }}}'
)
if ! response=$(curl -sK /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/outputs/so-manager_kafka" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" --fail 2>/dev/null); then
echo -e "\nFailed to force update to Elastic Fleet output policy for Kafka...\n"

4
salt/elasticsearch/ca.sls
View File

@@ -26,14 +26,14 @@ catrustscript:
GLOBALS: {{ GLOBALS }}
{% endif %}
cacertz:
elasticsearch_cacerts:
file.managed:
- name: /opt/so/conf/ca/cacerts
- source: salt://elasticsearch/cacerts
- user: 939
- group: 939
capemz:
elasticsearch_capems:
file.managed:
- name: /opt/so/conf/ca/tls-ca-bundle.pem
- source: salt://elasticsearch/tls-ca-bundle.pem

8
salt/elasticsearch/config.map.jinja
View File

@@ -6,8 +6,6 @@
{% from 'vars/globals.map.jinja' import GLOBALS %}
{% import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS with context %}
{% set HIGHLANDER = salt['pillar.get']('global:highlander', False) %}
{# this is a list of dicts containing hostname:ip for elasticsearch nodes that need to know about each other for cluster #}
{% set ELASTICSEARCH_SEED_HOSTS = [] %}
{% set node_data = salt['pillar.get']('elasticsearch:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %}
@@ -36,14 +34,8 @@
{% endfor %}
{% endif %}
{% elif grains.id.split('_') | last == 'searchnode' %}
{% if HIGHLANDER %}
{% do ELASTICSEARCHDEFAULTS.elasticsearch.config.node.roles.extend(['ml', 'master', 'transform']) %}
{% endif %}
{% do ELASTICSEARCHDEFAULTS.elasticsearch.config.update({'discovery': {'seed_hosts': [GLOBALS.manager]}}) %}
{% endif %}
{% if HIGHLANDER %}
{% do ELASTICSEARCHDEFAULTS.elasticsearch.config.xpack.ml.update({'enabled': true}) %}
{% endif %}
{% do ELASTICSEARCHDEFAULTS.elasticsearch.config.node.update({'name': GLOBALS.hostname}) %}
{% do ELASTICSEARCHDEFAULTS.elasticsearch.config.cluster.update({'name': GLOBALS.hostname}) %}

16
salt/elasticsearch/config.sls
View File

@@ -5,17 +5,12 @@
{% from 'allowed_states.map.jinja' import allowed_states %}
{% if sls.split('.')[0] in allowed_states %}
include:
- ssl
- elasticsearch.ca
{% from 'vars/globals.map.jinja' import GLOBALS %}
{% from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
vm.max_map_count:
sysctl.present:
- value: 262144
- value: {{ ELASTICSEARCHMERGED.vm.max_map_count }}
# Add ES Group
elasticsearchgroup:
@@ -103,10 +98,6 @@ esrolesdir:
- group: 939
- makedirs: True
eslibdir:
file.absent:
- name: /opt/so/conf/elasticsearch/lib
esingestdynamicconf:
file.recurse:
- name: /opt/so/conf/elasticsearch/ingest
@@ -124,11 +115,6 @@ esingestconf:
- group: 939
- show_changes: False
# Remove .fleet_final_pipeline-1 because we are using global@custom now
so-fleet-final-pipeline-remove:
file.absent:
- name: /opt/so/conf/elasticsearch/ingest/.fleet_final_pipeline-1
# Auto-generate Elasticsearch ingest node pipelines from pillar
{% for pipeline, config in ELASTICSEARCHMERGED.pipelines.items() %}
es_ingest_conf_{{pipeline}}:

165
salt/elasticsearch/defaults.yaml
View File

@@ -1,11 +1,15 @@
elasticsearch:
enabled: false
version: 8.18.8
version: 9.0.8
index_clean: true
vm:
max_map_count: 1048576
config:
action:
destructive_requires_name: true
cluster:
logsdb:
enabled: false
routing:
allocation:
disk:
@@ -115,7 +119,7 @@ elasticsearch:
ignore_missing_component_templates: []
index_patterns:
- so-case*
priority: 500
priority: 501
template:
mappings:
date_detection: false
@@ -127,8 +131,6 @@ elasticsearch:
match_mapping_type: string
settings:
index:
lifecycle:
name: so-case-logs
mapping:
total_fields:
limit: 1500
@@ -139,14 +141,7 @@ elasticsearch:
sort:
field: '@timestamp'
order: desc
policy:
phases:
hot:
actions: {}
min_age: 0ms
so-common:
close: 30
delete: 365
index_sorting: false
index_template:
composed_of:
@@ -210,7 +205,9 @@ elasticsearch:
- common-settings
- common-dynamic-mappings
- winlog-mappings
data_stream: {}
data_stream:
allow_custom_routing: false
hidden: false
ignore_missing_component_templates: []
index_patterns:
- logs-*-so*
@@ -270,7 +267,7 @@ elasticsearch:
ignore_missing_component_templates: []
index_patterns:
- so-detection*
priority: 500
priority: 501
template:
mappings:
date_detection: false
@@ -282,8 +279,6 @@ elasticsearch:
match_mapping_type: string
settings:
index:
lifecycle:
name: so-detection-logs
mapping:
total_fields:
limit: 1500
@@ -294,11 +289,6 @@ elasticsearch:
sort:
field: '@timestamp'
order: desc
policy:
phases:
hot:
actions: {}
min_age: 0ms
sos-backup:
index_sorting: false
index_template:
@@ -458,7 +448,7 @@ elasticsearch:
ignore_missing_component_templates: []
index_patterns:
- endgame*
priority: 500
priority: 501
template:
mappings:
date_detection: false
@@ -506,8 +496,6 @@ elasticsearch:
priority: 50
min_age: 30d
so-idh:
close: 30
delete: 365
index_sorting: false
index_template:
composed_of:
@@ -562,10 +550,13 @@ elasticsearch:
- dtc-user_agent-mappings
- common-settings
- common-dynamic-mappings
data_stream:
allow_custom_routing: false
hidden: false
ignore_missing_component_templates: []
index_patterns:
- so-idh-*
priority: 500
- logs-idh-so*
priority: 501
template:
mappings:
date_detection: false
@@ -675,11 +666,13 @@ elasticsearch:
- common-dynamic-mappings
- winlog-mappings
- hash-mappings
data_stream: {}
data_stream:
allow_custom_routing: false
hidden: false
ignore_missing_component_templates: []
index_patterns:
- logs-import-so*
priority: 500
priority: 501
template:
mappings:
date_detection: false
@@ -691,7 +684,6 @@ elasticsearch:
match_mapping_type: string
settings:
index:
final_pipeline: .fleet_final_pipeline-1
lifecycle:
name: so-import-logs
mapping:
@@ -735,7 +727,7 @@ elasticsearch:
ignore_missing_component_templates: []
index_patterns:
- so-ip*
priority: 500
priority: 501
template:
mappings:
date_detection: false
@@ -750,19 +742,12 @@ elasticsearch:
mapping:
total_fields:
limit: 1500
lifecycle:
name: so-ip-mappings-logs
number_of_replicas: 0
number_of_shards: 1
refresh_interval: 30s
sort:
field: '@timestamp'
order: desc
policy:
phases:
hot:
actions: {}
min_age: 0ms
so-items:
index_sorting: false
index_template:
@@ -771,7 +756,7 @@ elasticsearch:
ignore_missing_component_templates: []
index_patterns:
- .items-default-**
priority: 500
priority: 501
template:
mappings:
date_detection: false
@@ -850,60 +835,18 @@ elasticsearch:
priority: 50
min_age: 30d
so-kratos:
close: 30
delete: 365
index_sorting: false
index_template:
composed_of:
- agent-mappings
- dtc-agent-mappings
- base-mappings
- dtc-base-mappings
- client-mappings
- dtc-client-mappings
- container-mappings
- destination-mappings
- dtc-destination-mappings
- pb-override-destination-mappings
- dll-mappings
- dns-mappings
- dtc-dns-mappings
- ecs-mappings
- dtc-ecs-mappings
- error-mappings
- event-mappings
- dtc-event-mappings
- file-mappings
- dtc-file-mappings
- group-mappings
- host-mappings
- dtc-host-mappings
- http-mappings
- dtc-http-mappings
- log-mappings
- metadata-mappings
- network-mappings
- dtc-network-mappings
- observer-mappings
- dtc-observer-mappings
- organization-mappings
- package-mappings
- process-mappings
- dtc-process-mappings
- related-mappings
- rule-mappings
- dtc-rule-mappings
- server-mappings
- service-mappings
- dtc-service-mappings
- source-mappings
- dtc-source-mappings
- pb-override-source-mappings
- threat-mappings
- tls-mappings
- url-mappings
- user_agent-mappings
- dtc-user_agent-mappings
- common-settings
- common-dynamic-mappings
data_stream:
@@ -912,7 +855,7 @@ elasticsearch:
ignore_missing_component_templates: []
index_patterns:
- logs-kratos-so*
priority: 500
priority: 501
template:
mappings:
date_detection: false
@@ -960,8 +903,6 @@ elasticsearch:
priority: 50
min_age: 30d
so-hydra:
close: 30
delete: 365
index_sorting: false
index_template:
composed_of:
@@ -1022,7 +963,7 @@ elasticsearch:
ignore_missing_component_templates: []
index_patterns:
- logs-hydra-so*
priority: 500
priority: 501
template:
mappings:
date_detection: false
@@ -1077,7 +1018,7 @@ elasticsearch:
ignore_missing_component_templates: []
index_patterns:
- .lists-default-**
priority: 500
priority: 501
template:
mappings:
date_detection: false
@@ -1563,6 +1504,9 @@ elasticsearch:
- so-fleet_integrations.ip_mappings-1
- so-fleet_globals-1
- so-fleet_agent_id_verification-1
data_stream:
allow_custom_routing: false
hidden: false
ignore_missing_component_templates:
- logs-elastic_agent.cloudbeat@custom
index_patterns:
@@ -1798,6 +1742,9 @@ elasticsearch:
- so-fleet_integrations.ip_mappings-1
- so-fleet_globals-1
- so-fleet_agent_id_verification-1
data_stream:
allow_custom_routing: false
hidden: false
ignore_missing_component_templates:
- logs-elastic_agent.heartbeat@custom
index_patterns:
@@ -3057,8 +3004,6 @@ elasticsearch:
priority: 50
min_age: 30d
so-logs-soc:
close: 30
delete: 365
index_sorting: false
index_template:
composed_of:
@@ -3113,11 +3058,13 @@ elasticsearch:
- dtc-user_agent-mappings
- common-settings
- common-dynamic-mappings
data_stream: {}
data_stream:
allow_custom_routing: false
hidden: false
ignore_missing_component_templates: []
index_patterns:
- logs-soc-so*
priority: 500
priority: 501
template:
mappings:
date_detection: false
@@ -3707,10 +3654,13 @@ elasticsearch:
- vulnerability-mappings
- common-settings
- common-dynamic-mappings
data_stream:
allow_custom_routing: false
hidden: false
ignore_missing_component_templates: []
index_patterns:
- logs-logstash-default*
priority: 500
priority: 501
template:
mappings:
date_detection: false
@@ -4008,10 +3958,13 @@ elasticsearch:
- vulnerability-mappings
- common-settings
- common-dynamic-mappings
data_stream:
allow_custom_routing: false
hidden: false
ignore_missing_component_templates: []
index_patterns:
- logs-redis-default*
priority: 500
- logs-redis.log*
priority: 501
template:
mappings:
date_detection: false
@@ -4122,11 +4075,13 @@ elasticsearch:
- common-settings
- common-dynamic-mappings
- hash-mappings
data_stream: {}
data_stream:
allow_custom_routing: false
hidden: false
ignore_missing_component_templates: []
index_patterns:
- logs-strelka-so*
priority: 500
priority: 501
template:
mappings:
date_detection: false
@@ -4236,11 +4191,13 @@ elasticsearch:
- common-settings
- common-dynamic-mappings
- hash-mappings
data_stream: {}
data_stream:
allow_custom_routing: false
hidden: false
ignore_missing_component_templates: []
index_patterns:
- logs-suricata-so*
priority: 500
priority: 501
template:
mappings:
date_detection: false
@@ -4350,11 +4307,13 @@ elasticsearch:
- common-settings
- common-dynamic-mappings
- hash-mappings
data_stream: {}
data_stream:
allow_custom_routing: false
hidden: false
ignore_missing_component_templates: []
index_patterns:
- logs-suricata.alerts-*
priority: 500
priority: 501
template:
mappings:
date_detection: false
@@ -4464,11 +4423,13 @@ elasticsearch:
- vulnerability-mappings
- common-settings
- common-dynamic-mappings
data_stream: {}
data_stream:
allow_custom_routing: false
hidden: false
ignore_missing_component_templates: []
index_patterns:
- logs-syslog-so*
priority: 500
priority: 501
template:
mappings:
date_detection: false
@@ -4580,11 +4541,13 @@ elasticsearch:
- common-settings
- common-dynamic-mappings
- hash-mappings
data_stream: {}
data_stream:
allow_custom_routing: false
hidden: false
ignore_missing_component_templates: []
index_patterns:
- logs-zeek-so*
priority: 500
priority: 501
template:
mappings:
date_detection: false

52
salt/elasticsearch/enabled.sls
View File

@@ -6,7 +6,7 @@
{% from 'allowed_states.map.jinja' import allowed_states %}
{% if sls.split('.')[0] in allowed_states %}
{% from 'vars/globals.map.jinja' import GLOBALS %}
{% from 'docker/docker.map.jinja' import DOCKER %}
{% from 'docker/docker.map.jinja' import DOCKERMERGED %}
{% from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_NODES %}
{% from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_SEED_HOSTS %}
{% from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
@@ -14,6 +14,9 @@
{% from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS %}
include:
- ca
- elasticsearch.ca
- elasticsearch.ssl
- elasticsearch.config
- elasticsearch.sostatus
@@ -25,15 +28,15 @@ so-elasticsearch:
- user: elasticsearch
- networks:
- sobridge:
- ipv4_address: {{ DOCKER.containers['so-elasticsearch'].ip }}
- ipv4_address: {{ DOCKERMERGED.containers['so-elasticsearch'].ip }}
- extra_hosts:
{% for node in ELASTICSEARCH_NODES %}
{% for hostname, ip in node.items() %}
- {{hostname}}:{{ip}}
{% endfor %}
{% endfor %}
{% if DOCKER.containers['so-elasticsearch'].extra_hosts %}
{% for XTRAHOST in DOCKER.containers['so-elasticsearch'].extra_hosts %}
{% if DOCKERMERGED.containers['so-elasticsearch'].extra_hosts %}
{% for XTRAHOST in DOCKERMERGED.containers['so-elasticsearch'].extra_hosts %}
- {{ XTRAHOST }}
{% endfor %}
{% endif %}
@@ -42,17 +45,19 @@ so-elasticsearch:
- discovery.type=single-node
{% endif %}
- ES_JAVA_OPTS=-Xms{{ GLOBALS.elasticsearch.es_heap }} -Xmx{{ GLOBALS.elasticsearch.es_heap }} -Des.transport.cname_in_publish_address=true -Dlog4j2.formatMsgNoLookups=true
ulimits:
- memlock=-1:-1
- nofile=65536:65536
- nproc=4096
{% if DOCKER.containers['so-elasticsearch'].extra_env %}
{% for XTRAENV in DOCKER.containers['so-elasticsearch'].extra_env %}
{% if DOCKERMERGED.containers['so-elasticsearch'].extra_env %}
{% for XTRAENV in DOCKERMERGED.containers['so-elasticsearch'].extra_env %}
- {{ XTRAENV }}
{% endfor %}
{% endif %}
{% if DOCKERMERGED.containers['so-elasticsearch'].ulimits %}
- ulimits:
{% for ULIMIT in DOCKERMERGED.containers['so-elasticsearch'].ulimits %}
- {{ ULIMIT.name }}={{ ULIMIT.soft }}:{{ ULIMIT.hard }}
{% endfor %}
{% endif %}
- port_bindings:
{% for BINDING in DOCKER.containers['so-elasticsearch'].port_bindings %}
{% for BINDING in DOCKERMERGED.containers['so-elasticsearch'].port_bindings %}
- {{ BINDING }}
{% endfor %}
- binds:
@@ -61,11 +66,7 @@ so-elasticsearch:
- /nsm/elasticsearch:/usr/share/elasticsearch/data:rw
- /opt/so/log/elasticsearch:/var/log/elasticsearch:rw
- /opt/so/conf/ca/cacerts:/usr/share/elasticsearch/jdk/lib/security/cacerts:ro
{% if GLOBALS.is_manager %}
- /etc/pki/ca.crt:/usr/share/elasticsearch/config/ca.crt:ro
{% else %}
- /etc/pki/tls/certs/intca.crt:/usr/share/elasticsearch/config/ca.crt:ro
{% endif %}
- /etc/pki/elasticsearch.crt:/usr/share/elasticsearch/config/elasticsearch.crt:ro
- /etc/pki/elasticsearch.key:/usr/share/elasticsearch/config/elasticsearch.key:ro
- /etc/pki/elasticsearch.p12:/usr/share/elasticsearch/config/elasticsearch.p12:ro
@@ -76,28 +77,27 @@ so-elasticsearch:
- {{ repo }}:{{ repo }}:rw
{% endfor %}
{% endif %}
{% if DOCKER.containers['so-elasticsearch'].custom_bind_mounts %}
{% for BIND in DOCKER.containers['so-elasticsearch'].custom_bind_mounts %}
{% if DOCKERMERGED.containers['so-elasticsearch'].custom_bind_mounts %}
{% for BIND in DOCKERMERGED.containers['so-elasticsearch'].custom_bind_mounts %}
- {{ BIND }}
{% endfor %}
{% endif %}
- watch:
- file: cacertz
- file: trusttheca
- x509: elasticsearch_crt
- x509: elasticsearch_key
- file: elasticsearch_cacerts
- file: esyml
- require:
- file: trusttheca
- x509: elasticsearch_crt
- x509: elasticsearch_key
- file: elasticsearch_cacerts
- file: esyml
- file: eslog4jfile
- file: nsmesdir
- file: eslogdir
- file: cacertz
- x509: /etc/pki/elasticsearch.crt
- x509: /etc/pki/elasticsearch.key
- file: elasticp12perms
{% if GLOBALS.is_manager %}
- x509: pki_public_ca_crt
{% else %}
- x509: trusttheca
{% endif %}
- cmd: auth_users_roles_inode
- cmd: auth_users_inode

0
salt/elasticsearch/files/ingest-dynamic/.gitkeep Normal file
View File

16
salt/elasticsearch/files/ingest-dynamic/common → salt/elasticsearch/files/ingest/common
View File

@@ -1,5 +1,3 @@
{%- set HIGHLANDER = salt['pillar.get']('global:highlander', False) -%}
{%- raw -%}
{
"description" : "common",
"processors" : [
@@ -67,19 +65,7 @@
{ "append": { "if": "ctx.dataset_tag_temp != null", "field": "tags", "value": "{{dataset_tag_temp.1}}" } },
{ "grok": { "if": "ctx.http?.response?.status_code != null", "field": "http.response.status_code", "patterns": ["%{NUMBER:http.response.status_code:long} %{GREEDYDATA}"]} },
{ "set": { "if": "ctx?.metadata?.kafka != null" , "field": "kafka.id", "value": "{{metadata.kafka.partition}}{{metadata.kafka.offset}}{{metadata.kafka.timestamp}}", "ignore_failure": true } },
{ "remove": { "field": [ "message2", "type", "fields", "category", "module", "dataset", "dataset_tag_temp", "event.dataset_temp" ], "ignore_missing": true, "ignore_failure": true } }
{%- endraw %}
{%- if HIGHLANDER %}
,
{
"pipeline": {
"name": "ecs"
}
}
{%- endif %}
{%- raw %}
,
{ "remove": { "field": [ "message2", "type", "fields", "category", "module", "dataset", "dataset_tag_temp", "event.dataset_temp" ], "ignore_missing": true, "ignore_failure": true } },
{ "pipeline": { "name": "global@custom", "ignore_missing_pipeline": true, "description": "[Fleet] Global pipeline for all data streams" } }
]
}
{% endraw %}

241
salt/elasticsearch/files/ingest/global@custom
View File

@@ -1,31 +1,212 @@
{
"version": 3,
"_meta": {
"managed_by": "securityonion",
"managed": true
},
"description": "Custom pipeline for processing all incoming Fleet Agent documents. \n",
"processors": [
{ "set": { "ignore_failure": true, "field": "event.module", "value": "elastic_agent" } },
{ "split": { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "separator": "\\.", "target_field": "module_temp" } },
{ "split": { "if": "ctx.data_stream?.dataset != null && ctx.data_stream?.dataset.contains('.')", "field":"data_stream.dataset", "separator":"\\.", "target_field":"datastream_dataset_temp", "ignore_missing":true } },
{ "set": { "if": "ctx.module_temp != null", "override": true, "field": "event.module", "value": "{{module_temp.0}}" } },
{ "set": { "if": "ctx.datastream_dataset_temp != null && ctx.datastream_dataset_temp[0] == 'network_traffic'", "field":"event.module", "value":"{{ datastream_dataset_temp.0 }}", "ignore_failure":true, "ignore_empty_value":true, "description":"Fix EA network packet capture" } },
{ "gsub": { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "pattern": "^[^.]*.", "replacement": "", "target_field": "dataset_tag_temp" } },
{ "append": { "if": "ctx.dataset_tag_temp != null", "field": "tags", "value": "{{dataset_tag_temp}}", "allow_duplicates": false } },
{ "set": { "if": "ctx.network?.direction == 'egress'", "override": true, "field": "network.initiated", "value": "true" } },
{ "set": { "if": "ctx.network?.direction == 'ingress'", "override": true, "field": "network.initiated", "value": "false" } },
{ "set": { "if": "ctx.network?.type == 'ipv4'", "override": true, "field": "destination.ipv6", "value": "false" } },
{ "set": { "if": "ctx.network?.type == 'ipv6'", "override": true, "field": "destination.ipv6", "value": "true" } },
{ "set": { "if": "ctx.tags != null && ctx.tags.contains('import')", "override": true, "field": "data_stream.dataset", "value": "import" } },
{ "set": { "if": "ctx.tags != null && ctx.tags.contains('import')", "override": true, "field": "data_stream.namespace", "value": "so" } },
{ "community_id":{ "if": "ctx.event?.dataset == 'endpoint.events.network'", "ignore_failure":true } },
{ "set": { "if": "ctx.event?.module == 'fim'", "override": true, "field": "event.module", "value": "file_integrity" } },
{ "rename": { "if": "ctx.winlog?.provider_name == 'Microsoft-Windows-Windows Defender'", "ignore_missing": true, "field": "winlog.event_data.Threat Name", "target_field": "winlog.event_data.threat_name" } },
{ "set": { "if": "ctx?.metadata?.kafka != null" , "field": "kafka.id", "value": "{{metadata.kafka.partition}}{{metadata.kafka.offset}}{{metadata.kafka.timestamp}}", "ignore_failure": true } },
{ "set": { "if": "ctx.event?.dataset != null && ctx.event?.dataset == 'elasticsearch.server'", "field": "event.module", "value":"elasticsearch" }},
{"append": {"field":"related.ip","value":["{{source.ip}}","{{destination.ip}}"],"allow_duplicates":false,"if":"ctx?.event?.dataset == 'endpoint.events.network' && ctx?.source?.ip != null","ignore_failure":true}},
{"foreach": {"field":"host.ip","processor":{"append":{"field":"related.ip","value":"{{_ingest._value}}","allow_duplicates":false}},"if":"ctx?.event?.module == 'endpoint' && ctx?.host?.ip != null","ignore_missing":true, "description":"Extract IPs from Elastic Agent events (host.ip) and adds them to related.ip"}},
{ "remove": { "field": [ "message2", "type", "fields", "category", "module", "dataset", "event.dataset_temp", "dataset_tag_temp", "module_temp", "datastream_dataset_temp" ], "ignore_missing": true, "ignore_failure": true } }
]
}
"version": 3,
"_meta": {
"managed_by": "securityonion",
"managed": true
},
"description": "Custom pipeline for processing all incoming Fleet Agent documents. \n",
"processors": [
{
"set": {
"ignore_failure": true,
"field": "event.module",
"value": "elastic_agent"
}
},
{
"split": {
"if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')",
"field": "event.dataset",
"separator": "\\.",
"target_field": "module_temp"
}
},
{
"split": {
"if": "ctx.data_stream?.dataset != null && ctx.data_stream?.dataset.contains('.')",
"field": "data_stream.dataset",
"separator": "\\.",
"target_field": "datastream_dataset_temp",
"ignore_missing": true
}
},
{
"set": {
"if": "ctx.module_temp != null",
"override": true,
"field": "event.module",
"value": "{{module_temp.0}}"
}
},
{
"set": {
"if": "ctx.datastream_dataset_temp != null && ctx.datastream_dataset_temp[0] == 'network_traffic'",
"field": "event.module",
"value": "{{ datastream_dataset_temp.0 }}",
"ignore_failure": true,
"ignore_empty_value": true,
"description": "Fix EA network packet capture"
}
},
{
"gsub": {
"if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')",
"field": "event.dataset",
"pattern": "^[^.]*.",
"replacement": "",
"target_field": "dataset_tag_temp"
}
},
{
"append": {
"if": "ctx.dataset_tag_temp != null",
"field": "tags",
"value": "{{dataset_tag_temp}}",
"allow_duplicates": false
}
},
{
"set": {
"if": "ctx.network?.direction == 'egress'",
"override": true,
"field": "network.initiated",
"value": "true"
}
},
{
"set": {
"if": "ctx.network?.direction == 'ingress'",
"override": true,
"field": "network.initiated",
"value": "false"
}
},
{
"set": {
"if": "ctx.network?.type == 'ipv4'",
"override": true,
"field": "destination.ipv6",
"value": "false"
}
},
{
"set": {
"if": "ctx.network?.type == 'ipv6'",
"override": true,
"field": "destination.ipv6",
"value": "true"
}
},
{
"set": {
"if": "ctx.tags != null && ctx.tags.contains('import')",
"override": true,
"field": "data_stream.dataset",
"value": "import"
}
},
{
"set": {
"if": "ctx.tags != null && ctx.tags.contains('import')",
"override": true,
"field": "data_stream.namespace",
"value": "so"
}
},
{
"community_id": {
"if": "ctx.event?.dataset == 'endpoint.events.network'",
"ignore_failure": true
}
},
{
"set": {
"if": "ctx.event?.module == 'fim'",
"override": true,
"field": "event.module",
"value": "file_integrity"
}
},
{
"rename": {
"if": "ctx.winlog?.provider_name == 'Microsoft-Windows-Windows Defender'",
"ignore_missing": true,
"field": "winlog.event_data.Threat Name",
"target_field": "winlog.event_data.threat_name"
}
},
{
"set": {
"if": "ctx?.metadata?.kafka != null",
"field": "kafka.id",
"value": "{{metadata.kafka.partition}}{{metadata.kafka.offset}}{{metadata.kafka.timestamp}}",
"ignore_failure": true
}
},
{
"set": {
"if": "ctx.event?.dataset != null && ctx.event?.dataset == 'elasticsearch.server'",
"field": "event.module",
"value": "elasticsearch"
}
},
{
"append": {
"field": "related.ip",
"value": [
"{{source.ip}}",
"{{destination.ip}}"
],
"allow_duplicates": false,
"if": "ctx?.event?.dataset == 'endpoint.events.network' && ctx?.source?.ip != null",
"ignore_failure": true
}
},
{
"foreach": {
"field": "host.ip",
"processor": {
"append": {
"field": "related.ip",
"value": "{{_ingest._value}}",
"allow_duplicates": false
}
},
"if": "ctx?.event?.module == 'endpoint' && ctx?.host?.ip != null",
"ignore_missing": true,
"description": "Extract IPs from Elastic Agent events (host.ip) and adds them to related.ip"
}
},
{
"pipeline": {
"name": ".fleet_final_pipeline-1",
"ignore_missing_pipeline": true
}
},
{
"remove": {
"field": "event.agent_id_status",
"ignore_missing": true,
"if": "ctx?.event?.agent_id_status == 'auth_metadata_missing'"
}
},
{
"remove": {
"field": [
"message2",
"type",
"fields",
"category",
"module",
"dataset",
"event.dataset_temp",
"dataset_tag_temp",
"module_temp",
"datastream_dataset_temp"
],
"ignore_missing": true,
"ignore_failure": true
}
}
]
}

103
salt/elasticsearch/files/ingest/kratos
View File

@@ -1,9 +1,98 @@
{
"description" : "kratos",
"processors" : [
{"set":{"field":"audience","value":"access","override":false,"ignore_failure":true}},
{"set":{"field":"event.dataset","ignore_empty_value":true,"ignore_failure":true,"value":"kratos.{{{audience}}}","media_type":"text/plain"}},
{"set":{"field":"event.action","ignore_failure":true,"copy_from":"msg" }},
{ "pipeline": { "name": "common" } }
]
"description": "kratos",
"processors": [
{
"set": {
"field": "audience",
"value": "access",
"override": false,
"ignore_failure": true
}
},
{
"set": {
"field": "event.dataset",
"ignore_empty_value": true,
"ignore_failure": true,
"value": "kratos.{{{audience}}}",
"media_type": "text/plain"
}
},
{
"set": {
"field": "event.action",
"ignore_failure": true,
"copy_from": "msg"
}
},
{
"rename": {
"field": "http_request",
"target_field": "http.request",
"ignore_failure": true,
"ignore_missing": true
}
},
{
"rename": {
"field": "http_response",
"target_field": "http.response",
"ignore_failure": true,
"ignore_missing": true
}
},
{
"rename": {
"field": "http.request.path",
"target_field": "http.uri",
"ignore_failure": true,
"ignore_missing": true
}
},
{
"rename": {
"field": "http.request.method",
"target_field": "http.method",
"ignore_failure": true,
"ignore_missing": true
}
},
{
"rename": {
"field": "http.request.method",
"target_field": "http.method",
"ignore_failure": true,
"ignore_missing": true
}
},
{
"rename": {
"field": "http.request.query",
"target_field": "http.query",
"ignore_failure": true,
"ignore_missing": true
}
},
{
"rename": {
"field": "http.request.headers.user-agent",
"target_field": "http.useragent",
"ignore_failure": true,
"ignore_missing": true
}
},
{
"rename": {
"field": "file",
"target_field": "file.path",
"ignore_failure": true,
"ignore_missing": true
}
},
{
"pipeline": {
"name": "common"
}
}
]
}

6
salt/elasticsearch/files/ingest/suricata.common
View File

@@ -22,6 +22,12 @@
"ignore_failure": true
}
},
{
"lowercase": {
"field": "network.transport",
"ignore_failure": true
}
},
{
"rename": {
"field": "message2.in_iface",

Some files were not shown because too many files have changed in this diff Show More