From e1e8a20e11aba12ddc83e8559d88904b7a4db4b9 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 10 Mar 2022 17:09:00 -0500 Subject: [PATCH 01/12] make sure values exist in data structure --- salt/elasticsearch/template.map.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/template.map.jinja b/salt/elasticsearch/template.map.jinja index 8168fec50..77c255c4c 100644 --- a/salt/elasticsearch/template.map.jinja +++ b/salt/elasticsearch/template.map.jinja @@ -1,7 +1,7 @@ {% import_yaml 'elasticsearch/defaults.yaml' as ESCONFIG with context %} {%- set ES_INDEX_SETTINGS = salt['pillar.get']('elasticsearch:index_settings', default=ESCONFIG.elasticsearch.index_settings, merge=True) %} {% for index, settings in ES_INDEX_SETTINGS.items() %} - {% if settings.index_sorting, False %} + {% if not settings.get('index_sorting', False) | to_bool and settings.index_template.template.settings.index.sort is defined %} {% do settings.index_template.template.settings.index.pop('sort') %} {% endif %} {% endfor %} From 8e43a6e571cbc2ae21d16fb5befc5633cb831c21 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Fri, 11 Mar 2022 16:22:06 +0000 Subject: [PATCH 02/12] Don't generate index template if index_template definition is not present in pillar --- salt/elasticsearch/init.sls | 2 ++ salt/elasticsearch/template.map.jinja | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index 19f1aeacb..1d1518b3c 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -207,6 +207,7 @@ escomponenttemplates: # Auto-generate templates from defaults file {% for index, settings in ES_INDEX_SETTINGS.items() %} + {% if settings.index_template is defined %} es_index_template_{{index}}: file.managed: - name: /opt/so/conf/elasticsearch/templates/index/{{ index }}-template.json @@ -216,6 +217,7 @@ es_index_template_{{index}}: - template: jinja - onchanges_in: - cmd: so-elasticsearch-templates + {% endif %} {% endfor %} {% if TEMPLATES %} diff --git a/salt/elasticsearch/template.map.jinja b/salt/elasticsearch/template.map.jinja index 77c255c4c..9c90cc28f 100644 --- a/salt/elasticsearch/template.map.jinja +++ b/salt/elasticsearch/template.map.jinja @@ -1,7 +1,9 @@ {% import_yaml 'elasticsearch/defaults.yaml' as ESCONFIG with context %} {%- set ES_INDEX_SETTINGS = salt['pillar.get']('elasticsearch:index_settings', default=ESCONFIG.elasticsearch.index_settings, merge=True) %} {% for index, settings in ES_INDEX_SETTINGS.items() %} - {% if not settings.get('index_sorting', False) | to_bool and settings.index_template.template.settings.index.sort is defined %} - {% do settings.index_template.template.settings.index.pop('sort') %} + {% if settings.index_template is defined %} + {% if not settings.get('index_sorting', False) | to_bool and settings.index_template.template.settings.index.sort is defined %} + {% do settings.index_template.template.settings.index.pop('sort') %} + {% endif %} {% endif %} {% endfor %} From dc258cf04332f14dff9b7b25f51741dd1b4cd8ce Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Fri, 11 Mar 2022 16:22:55 +0000 Subject: [PATCH 03/12] Load custom component templates in so-elasticsearch-templates-load --- .../tools/sbin/so-elasticsearch-templates-load | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load b/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load index f3bcaa308..f61228c3a 100755 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load @@ -60,6 +60,15 @@ echo "Loading Security Onion component templates..." for i in *; do TEMPLATE=$(echo $i | cut -d '.' -f1); echo "$TEMPLATE"; {{ ELASTICCURL }} -k ${ELASTICSEARCH_AUTH} -s -XPUT -L https://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_component_template/$TEMPLATE -H 'Content-Type: application/json' -d@$i 2>/dev/null; echo; done echo +if [ -d ${ELASTICSEARCH_TEMPLATES}/component/custom ]; then + # Load custom component templates + cd ${ELASTICSEARCH_TEMPLATES}/component/custom + + echo "Loading custom component templates..." + for i in *; do TEMPLATE=$(echo $i | cut -d '.' -f1); echo "$TEMPLATE"; curl -K /opt/so/conf/elasticsearch/curl.config -k ${ELASTICSEARCH_AUTH} -s -XPUT -L https://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_component_template/$TEMPLATE -H 'Content-Type: application/json' -d@$i 2>/dev/null; echo; done + echo +fi + # Load SO index templates cd ${ELASTICSEARCH_TEMPLATES}/index From d12ff503c27f370f2150b929acfa277d6ab68dac Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Fri, 11 Mar 2022 16:23:19 +0000 Subject: [PATCH 04/12] Chage role loading verbiage --- salt/elasticsearch/tools/sbin/so-elasticsearch-roles-load | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-roles-load b/salt/elasticsearch/tools/sbin/so-elasticsearch-roles-load index 2da8f85e4..c407ac183 100755 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-roles-load +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-roles-load @@ -48,7 +48,7 @@ fi cd ${ELASTICSEARCH_ROLES} -echo "Loading templates..." +echo "Loading roles..." for role in *; do name=$(echo "$role" | cut -d. -f1) so-elasticsearch-query _security/role/$name -XPUT -d @"$role" From 70ed20f6918cfd62d788c093eca98b495ab639c0 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Fri, 11 Mar 2022 18:07:23 +0000 Subject: [PATCH 05/12] Add new sls file for custom ES index templates --- pillar/elasticsearch/index_templates.sls | 2 ++ pillar/top.sls | 12 ++++++------ 2 files changed, 8 insertions(+), 6 deletions(-) create mode 100644 pillar/elasticsearch/index_templates.sls diff --git a/pillar/elasticsearch/index_templates.sls b/pillar/elasticsearch/index_templates.sls new file mode 100644 index 000000000..a02a1818c --- /dev/null +++ b/pillar/elasticsearch/index_templates.sls @@ -0,0 +1,2 @@ +elasticsearch: + index_settings: diff --git a/pillar/top.sls b/pillar/top.sls index 8ab666d0d..e96892684 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -15,12 +15,12 @@ base: - logstash - logstash.manager - logstash.search - - elasticsearch.search + - elasticsearch.index_templates '*_manager': - logstash - logstash.manager - - elasticsearch.manager + - elasticsearch.index_templates '*_manager or *_managersearch': - match: compound @@ -46,7 +46,7 @@ base: - zeeklogs - secrets - healthcheck.eval - - elasticsearch.eval + - elasticsearch.index_templates {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %} - elasticsearch.auth {% endif %} @@ -60,7 +60,7 @@ base: - logstash - logstash.manager - logstash.search - - elasticsearch.search + - elasticsearch.inedx_templates {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %} - elasticsearch.auth {% endif %} @@ -106,7 +106,7 @@ base: '*_searchnode': - logstash - logstash.search - - elasticsearch.search + - elasticsearch.index_templates - elasticsearch.auth - global - minions.{{ grains.id }} @@ -122,7 +122,7 @@ base: '*_import': - zeeklogs - secrets - - elasticsearch.eval + - elasticsearch.index_templates {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %} - elasticsearch.auth {% endif %} From ba24f7589339f4736de0aa9cbd86df0ecdbe1668 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Fri, 11 Mar 2022 18:11:16 +0000 Subject: [PATCH 06/12] Fix index typo --- pillar/top.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pillar/top.sls b/pillar/top.sls index e96892684..097f5b108 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -60,7 +60,7 @@ base: - logstash - logstash.manager - logstash.search - - elasticsearch.inedx_templates + - elasticsearch.index_templates {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %} - elasticsearch.auth {% endif %} From 8ebeeb497f516f5d9fa43c1c98759a067a7fcbf8 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 15 Mar 2022 09:43:09 -0400 Subject: [PATCH 07/12] add configuration to override leak detector defaults --- .github/workflows/leaktest.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/leaktest.yml b/.github/workflows/leaktest.yml index 590e220d0..c2f7f8010 100644 --- a/.github/workflows/leaktest.yml +++ b/.github/workflows/leaktest.yml @@ -13,3 +13,5 @@ jobs: - name: Gitleaks uses: zricethezav/gitleaks-action@master + with: + config-path: .github/.gitleaks.toml From cec9cba40e6e5ad10fbdc19048822eb5343e2176 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 15 Mar 2022 09:47:57 -0400 Subject: [PATCH 08/12] Create .gitleaks.toml --- .github/.gitleaks.toml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .github/.gitleaks.toml diff --git a/.github/.gitleaks.toml b/.github/.gitleaks.toml new file mode 100644 index 000000000..68ca574d6 --- /dev/null +++ b/.github/.gitleaks.toml @@ -0,0 +1,3 @@ +[allowlist] +description = "ignore enteprise-attack.json since it has false positives due to Twitter IDs" +paths = ['''salt/nginx/files/enterprise-attack.json'''] From b810f144289910bbd798f7e64fa4b8c6ef443f4f Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 15 Mar 2022 09:53:11 -0400 Subject: [PATCH 09/12] Update .gitleaks.toml --- .github/.gitleaks.toml | 547 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 545 insertions(+), 2 deletions(-) diff --git a/.github/.gitleaks.toml b/.github/.gitleaks.toml index 68ca574d6..312e46d97 100644 --- a/.github/.gitleaks.toml +++ b/.github/.gitleaks.toml @@ -1,3 +1,546 @@ +title = "gitleaks config" + +# Gitleaks rules are defined by regular expressions and entropy ranges. +# Some secrets have unique signatures which make detecting those secrets easy. +# Examples of those secrets would be GitLab Personal Access Tokens, AWS keys, and GitHub Access Tokens. +# All these examples have defined prefixes like `glpat`, `AKIA`, `ghp_`, etc. +# +# Other secrets might just be a hash which means we need to write more complex rules to verify +# that what we are matching is a secret. +# +# Here is an example of a semi-generic secret +# +# discord_client_secret = "8dyfuiRyq=vVc3RRr_edRk-fK__JItpZ" +# +# We can write a regular expression to capture the variable name (identifier), +# the assignment symbol (like '=' or ':='), and finally the actual secret. +# The structure of a rule to match this example secret is below: +# +# Beginning string +# quotation +# │ End string quotation +# │ │ +# ▼ ▼ +# (?i)(discord[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9=_\-]{32})['\"] +# +# ▲ ▲ ▲ +# │ │ │ +# │ │ │ +# identifier assignment symbol +# Secret +# +[[rules]] +id = "gitlab-pat" +description = "GitLab Personal Access Token" +regex = '''glpat-[0-9a-zA-Z\-\_]{20}''' + +[[rules]] +id = "aws-access-token" +description = "AWS" +regex = '''(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}''' + +# Cryptographic keys +[[rules]] +id = "PKCS8-PK" +description = "PKCS8 private key" +regex = '''-----BEGIN PRIVATE KEY-----''' + +[[rules]] +id = "RSA-PK" +description = "RSA private key" +regex = '''-----BEGIN RSA PRIVATE KEY-----''' + +[[rules]] +id = "OPENSSH-PK" +description = "SSH private key" +regex = '''-----BEGIN OPENSSH PRIVATE KEY-----''' + +[[rules]] +id = "PGP-PK" +description = "PGP private key" +regex = '''-----BEGIN PGP PRIVATE KEY BLOCK-----''' + +[[rules]] +id = "github-pat" +description = "GitHub Personal Access Token" +regex = '''ghp_[0-9a-zA-Z]{36}''' + +[[rules]] +id = "github-oauth" +description = "GitHub OAuth Access Token" +regex = '''gho_[0-9a-zA-Z]{36}''' + +[[rules]] +id = "SSH-DSA-PK" +description = "SSH (DSA) private key" +regex = '''-----BEGIN DSA PRIVATE KEY-----''' + +[[rules]] +id = "SSH-EC-PK" +description = "SSH (EC) private key" +regex = '''-----BEGIN EC PRIVATE KEY-----''' + + +[[rules]] +id = "github-app-token" +description = "GitHub App Token" +regex = '''(ghu|ghs)_[0-9a-zA-Z]{36}''' + +[[rules]] +id = "github-refresh-token" +description = "GitHub Refresh Token" +regex = '''ghr_[0-9a-zA-Z]{76}''' + +[[rules]] +id = "shopify-shared-secret" +description = "Shopify shared secret" +regex = '''shpss_[a-fA-F0-9]{32}''' + +[[rules]] +id = "shopify-access-token" +description = "Shopify access token" +regex = '''shpat_[a-fA-F0-9]{32}''' + +[[rules]] +id = "shopify-custom-access-token" +description = "Shopify custom app access token" +regex = '''shpca_[a-fA-F0-9]{32}''' + +[[rules]] +id = "shopify-private-app-access-token" +description = "Shopify private app access token" +regex = '''shppa_[a-fA-F0-9]{32}''' + +[[rules]] +id = "slack-access-token" +description = "Slack token" +regex = '''xox[baprs]-([0-9a-zA-Z]{10,48})?''' + +[[rules]] +id = "stripe-access-token" +description = "Stripe" +regex = '''(?i)(sk|pk)_(test|live)_[0-9a-z]{10,32}''' + +[[rules]] +id = "pypi-upload-token" +description = "PyPI upload token" +regex = '''pypi-AgEIcHlwaS5vcmc[A-Za-z0-9\-_]{50,1000}''' + +[[rules]] +id = "gcp-service-account" +description = "Google (GCP) Service-account" +regex = '''\"type\": \"service_account\"''' + +[[rules]] +id = "heroku-api-key" +description = "Heroku API Key" +regex = ''' (?i)(heroku[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([0-9A-F]{8}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{12})['\"]''' +secretGroup = 3 + +[[rules]] +id = "slack-web-hook" +description = "Slack Webhook" +regex = '''https://hooks.slack.com/services/T[a-zA-Z0-9_]{8}/B[a-zA-Z0-9_]{8,12}/[a-zA-Z0-9_]{24}''' + +[[rules]] +id = "twilio-api-key" +description = "Twilio API Key" +regex = '''SK[0-9a-fA-F]{32}''' + +[[rules]] +id = "age-secret-key" +description = "Age secret key" +regex = '''AGE-SECRET-KEY-1[QPZRY9X8GF2TVDW0S3JN54KHCE6MUA7L]{58}''' + +[[rules]] +id = "facebook-token" +description = "Facebook token" +regex = '''(?i)(facebook[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-f0-9]{32})['\"]''' +secretGroup = 3 + +[[rules]] +id = "twitter-token" +description = "Twitter token" +regex = '''(?i)(twitter[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-f0-9]{35,44})['\"]''' +secretGroup = 3 + +[[rules]] +id = "adobe-client-id" +description = "Adobe Client ID (Oauth Web)" +regex = '''(?i)(adobe[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-f0-9]{32})['\"]''' +secretGroup = 3 + +[[rules]] +id = "adobe-client-secret" +description = "Adobe Client Secret" +regex = '''(p8e-)(?i)[a-z0-9]{32}''' + +[[rules]] +id = "alibaba-access-key-id" +description = "Alibaba AccessKey ID" +regex = '''(LTAI)(?i)[a-z0-9]{20}''' + +[[rules]] +id = "alibaba-secret-key" +description = "Alibaba Secret Key" +regex = '''(?i)(alibaba[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{30})['\"]''' +secretGroup = 3 + +[[rules]] +id = "asana-client-id" +description = "Asana Client ID" +regex = '''(?i)(asana[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([0-9]{16})['\"]''' +secretGroup = 3 + +[[rules]] +id = "asana-client-secret" +description = "Asana Client Secret" +regex = '''(?i)(asana[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{32})['\"]''' +secretGroup = 3 + +[[rules]] +id = "atlassian-api-token" +description = "Atlassian API token" +regex = '''(?i)(atlassian[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{24})['\"]''' +secretGroup = 3 + +[[rules]] +id = "bitbucket-client-id" +description = "Bitbucket client ID" +regex = '''(?i)(bitbucket[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{32})['\"]''' +secretGroup = 3 + +[[rules]] +id = "bitbucket-client-secret" +description = "Bitbucket client secret" +regex = '''(?i)(bitbucket[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9_\-]{64})['\"]''' +secretGroup = 3 + +[[rules]] +id = "beamer-api-token" +description = "Beamer API token" +regex = '''(?i)(beamer[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](b_[a-z0-9=_\-]{44})['\"]''' +secretGroup = 3 + +[[rules]] +id = "clojars-api-token" +description = "Clojars API token" +regex = '''(CLOJARS_)(?i)[a-z0-9]{60}''' + +[[rules]] +id = "contentful-delivery-api-token" +description = "Contentful delivery API token" +regex = '''(?i)(contentful[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9\-=_]{43})['\"]''' +secretGroup = 3 + +[[rules]] +id = "databricks-api-token" +description = "Databricks API token" +regex = '''dapi[a-h0-9]{32}''' + +[[rules]] +id = "discord-api-token" +description = "Discord API key" +regex = '''(?i)(discord[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-h0-9]{64})['\"]''' +secretGroup = 3 + +[[rules]] +id = "discord-client-id" +description = "Discord client ID" +regex = '''(?i)(discord[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([0-9]{18})['\"]''' +secretGroup = 3 + +[[rules]] +id = "discord-client-secret" +description = "Discord client secret" +regex = '''(?i)(discord[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9=_\-]{32})['\"]''' +secretGroup = 3 + +[[rules]] +id = "doppler-api-token" +description = "Doppler API token" +regex = '''['\"](dp\.pt\.)(?i)[a-z0-9]{43}['\"]''' + +[[rules]] +id = "dropbox-api-secret" +description = "Dropbox API secret/key" +regex = '''(?i)(dropbox[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{15})['\"]''' + +[[rules]] +id = "dropbox--api-key" +description = "Dropbox API secret/key" +regex = '''(?i)(dropbox[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{15})['\"]''' + +[[rules]] +id = "dropbox-short-lived-api-token" +description = "Dropbox short lived API token" +regex = '''(?i)(dropbox[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](sl\.[a-z0-9\-=_]{135})['\"]''' + +[[rules]] +id = "dropbox-long-lived-api-token" +description = "Dropbox long lived API token" +regex = '''(?i)(dropbox[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"][a-z0-9]{11}(AAAAAAAAAA)[a-z0-9\-_=]{43}['\"]''' + +[[rules]] +id = "duffel-api-token" +description = "Duffel API token" +regex = '''['\"]duffel_(test|live)_(?i)[a-z0-9_-]{43}['\"]''' + +[[rules]] +id = "dynatrace-api-token" +description = "Dynatrace API token" +regex = '''['\"]dt0c01\.(?i)[a-z0-9]{24}\.[a-z0-9]{64}['\"]''' + +[[rules]] +id = "easypost-api-token" +description = "EasyPost API token" +regex = '''['\"]EZAK(?i)[a-z0-9]{54}['\"]''' + +[[rules]] +id = "easypost-test-api-token" +description = "EasyPost test API token" +regex = '''['\"]EZTK(?i)[a-z0-9]{54}['\"]''' + +[[rules]] +id = "fastly-api-token" +description = "Fastly API token" +regex = '''(?i)(fastly[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9\-=_]{32})['\"]''' +secretGroup = 3 + +[[rules]] +id = "finicity-client-secret" +description = "Finicity client secret" +regex = '''(?i)(finicity[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{20})['\"]''' +secretGroup = 3 + +[[rules]] +id = "finicity-api-token" +description = "Finicity API token" +regex = '''(?i)(finicity[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-f0-9]{32})['\"]''' +secretGroup = 3 + +[[rules]] +id = "flutterwave-public-key" +description = "Flutterwave public key" +regex = '''FLWPUBK_TEST-(?i)[a-h0-9]{32}-X''' + +[[rules]] +id = "flutterwave-secret-key" +description = "Flutterwave secret key" +regex = '''FLWSECK_TEST-(?i)[a-h0-9]{32}-X''' + +[[rules]] +id = "flutterwave-enc-key" +description = "Flutterwave encrypted key" +regex = '''FLWSECK_TEST[a-h0-9]{12}''' + +[[rules]] +id = "frameio-api-token" +description = "Frame.io API token" +regex = '''fio-u-(?i)[a-z0-9\-_=]{64}''' + +[[rules]] +id = "gocardless-api-token" +description = "GoCardless API token" +regex = '''['\"]live_(?i)[a-z0-9\-_=]{40}['\"]''' + +[[rules]] +id = "grafana-api-token" +description = "Grafana API token" +regex = '''['\"]eyJrIjoi(?i)[a-z0-9\-_=]{72,92}['\"]''' + +[[rules]] +id = "hashicorp-tf-api-token" +description = "HashiCorp Terraform user/org API token" +regex = '''['\"](?i)[a-z0-9]{14}\.atlasv1\.[a-z0-9\-_=]{60,70}['\"]''' + +[[rules]] +id = "hubspot-api-token" +description = "HubSpot API token" +regex = '''(?i)(hubspot[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-h0-9]{8}-[a-h0-9]{4}-[a-h0-9]{4}-[a-h0-9]{4}-[a-h0-9]{12})['\"]''' +secretGroup = 3 + +[[rules]] +id = "intercom-api-token" +description = "Intercom API token" +regex = '''(?i)(intercom[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9=_]{60})['\"]''' +secretGroup = 3 + +[[rules]] +id = "intercom-client-secret" +description = "Intercom client secret/ID" +regex = '''(?i)(intercom[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-h0-9]{8}-[a-h0-9]{4}-[a-h0-9]{4}-[a-h0-9]{4}-[a-h0-9]{12})['\"]''' +secretGroup = 3 + +[[rules]] +id = "ionic-api-token" +description = "Ionic API token" +regex = '''(?i)(ionic[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](ion_[a-z0-9]{42})['\"]''' + +[[rules]] +id = "linear-api-token" +description = "Linear API token" +regex = '''lin_api_(?i)[a-z0-9]{40}''' + +[[rules]] +id = "linear-client-secret" +description = "Linear client secret/ID" +regex = '''(?i)(linear[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-f0-9]{32})['\"]''' +secretGroup = 3 + +[[rules]] +id = "lob-api-key" +description = "Lob API Key" +regex = '''(?i)(lob[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]((live|test)_[a-f0-9]{35})['\"]''' +secretGroup = 3 + +[[rules]] +id = "lob-pub-api-key" +description = "Lob Publishable API Key" +regex = '''(?i)(lob[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]((test|live)_pub_[a-f0-9]{31})['\"]''' +secretGroup = 3 + +[[rules]] +id = "mailchimp-api-key" +description = "Mailchimp API key" +regex = '''(?i)(mailchimp[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-f0-9]{32}-us20)['\"]''' +secretGroup = 3 + +[[rules]] +id = "mailgun-private-api-token" +description = "Mailgun private API token" +regex = '''(?i)(mailgun[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](key-[a-f0-9]{32})['\"]''' +secretGroup = 3 + +[[rules]] +id = "mailgun-pub-key" +description = "Mailgun public validation key" +regex = '''(?i)(mailgun[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](pubkey-[a-f0-9]{32})['\"]''' +secretGroup = 3 + +[[rules]] +id = "mailgun-signing-key" +description = "Mailgun webhook signing key" +regex = '''(?i)(mailgun[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-h0-9]{32}-[a-h0-9]{8}-[a-h0-9]{8})['\"]''' +secretGroup = 3 + +[[rules]] +id = "mapbox-api-token" +description = "Mapbox API token" +regex = '''(?i)(pk\.[a-z0-9]{60}\.[a-z0-9]{22})''' + +[[rules]] +id = "messagebird-api-token" +description = "MessageBird API token" +regex = '''(?i)(messagebird[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{25})['\"]''' +secretGroup = 3 + +[[rules]] +id = "messagebird-client-id" +description = "MessageBird API client ID" +regex = '''(?i)(messagebird[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-h0-9]{8}-[a-h0-9]{4}-[a-h0-9]{4}-[a-h0-9]{4}-[a-h0-9]{12})['\"]''' +secretGroup = 3 + +[[rules]] +id = "new-relic-user-api-key" +description = "New Relic user API Key" +regex = '''['\"](NRAK-[A-Z0-9]{27})['\"]''' + +[[rules]] +id = "new-relic-user-api-id" +description = "New Relic user API ID" +regex = '''(?i)(newrelic[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([A-Z0-9]{64})['\"]''' +secretGroup = 3 + +[[rules]] +id = "new-relic-browser-api-token" +description = "New Relic ingest browser API token" +regex = '''['\"](NRJS-[a-f0-9]{19})['\"]''' + +[[rules]] +id = "npm-access-token" +description = "npm access token" +regex = '''['\"](npm_(?i)[a-z0-9]{36})['\"]''' + +[[rules]] +id = "planetscale-password" +description = "PlanetScale password" +regex = '''pscale_pw_(?i)[a-z0-9\-_\.]{43}''' + +[[rules]] +id = "planetscale-api-token" +description = "PlanetScale API token" +regex = '''pscale_tkn_(?i)[a-z0-9\-_\.]{43}''' + +[[rules]] +id = "postman-api-token" +description = "Postman API token" +regex = '''PMAK-(?i)[a-f0-9]{24}\-[a-f0-9]{34}''' + +[[rules]] +id = "pulumi-api-token" +description = "Pulumi API token" +regex = '''pul-[a-f0-9]{40}''' + +[[rules]] +id = "rubygems-api-token" +description = "Rubygem API token" +regex = '''rubygems_[a-f0-9]{48}''' + +[[rules]] +id = "sendgrid-api-token" +description = "SendGrid API token" +regex = '''SG\.(?i)[a-z0-9_\-\.]{66}''' + +[[rules]] +id = "sendinblue-api-token" +description = "Sendinblue API token" +regex = '''xkeysib-[a-f0-9]{64}\-(?i)[a-z0-9]{16}''' + +[[rules]] +id = "shippo-api-token" +description = "Shippo API token" +regex = '''shippo_(live|test)_[a-f0-9]{40}''' + +[[rules]] +id = "linkedin-client-secret" +description = "LinkedIn Client secret" +regex = '''(?i)(linkedin[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z]{16})['\"]''' +secretGroup = 3 + +[[rules]] +id = "linkedin-client-id" +description = "LinkedIn Client ID" +regex = '''(?i)(linkedin[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{14})['\"]''' +secretGroup = 3 + +[[rules]] +id = "twitch-api-token" +description = "Twitch API token" +regex = '''(?i)(twitch[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{30})['\"]''' +secretGroup = 3 + +[[rules]] +id = "typeform-api-token" +description = "Typeform API token" +regex = '''(?i)(typeform[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}(tfp_[a-z0-9\-_\.=]{59})''' +secretGroup = 3 + +[[rules]] +id = "generic-api-key" +description = "Generic API Key" +regex = '''(?i)((key|api[^Version]|token|secret|password)[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([0-9a-zA-Z\-_=]{8,64})['\"]''' +entropy = 3.7 +secretGroup = 4 + + [allowlist] -description = "ignore enteprise-attack.json since it has false positives due to Twitter IDs" -paths = ['''salt/nginx/files/enterprise-attack.json'''] +description = "global allow lists" +regexes = ['''219-09-9999''', '''078-05-1120''', '''(9[0-9]{2}|666)-\d{2}-\d{4}'''] +paths = [ + '''gitleaks.toml''', + '''(.*?)(jpg|gif|doc|pdf|bin|svg|socket)$''', + '''(go.mod|go.sum)$''' + + '''salt/nginx/files/enterprise-attack.json''' +] From 408f9d66956e207621dca3908ad8f4f95ccbcf25 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 15 Mar 2022 09:53:27 -0400 Subject: [PATCH 10/12] Update .gitleaks.toml --- .github/.gitleaks.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/.gitleaks.toml b/.github/.gitleaks.toml index 312e46d97..3ac46149b 100644 --- a/.github/.gitleaks.toml +++ b/.github/.gitleaks.toml @@ -540,7 +540,7 @@ regexes = ['''219-09-9999''', '''078-05-1120''', '''(9[0-9]{2}|666)-\d{2}-\d{4}' paths = [ '''gitleaks.toml''', '''(.*?)(jpg|gif|doc|pdf|bin|svg|socket)$''', - '''(go.mod|go.sum)$''' + '''(go.mod|go.sum)$''', '''salt/nginx/files/enterprise-attack.json''' ] From 5f56c7a261f1c477982ef455f3f13faf31092a31 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Tue, 15 Mar 2022 14:32:00 +0000 Subject: [PATCH 11/12] Replace ELASTICCURL with so-elasticsearch-query --- .../tools/sbin/so-elasticsearch-templates-load | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load b/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load index f61228c3a..1b6ac4685 100755 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load @@ -50,30 +50,20 @@ fi cd ${ELASTICSEARCH_TEMPLATES}/component/ecs echo "Loading ECS component templates..." -for i in *; do TEMPLATE=$(echo $i | cut -d '.' -f1); echo "$TEMPLATE-mappings"; {{ ELASTICCURL }} -k ${ELASTICSEARCH_AUTH} -s -XPUT -L https://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_component_template/$TEMPLATE-mappings -H 'Content-Type: application/json' -d@$i 2>/dev/null; echo; done -echo +for i in *; do TEMPLATE=$(echo $i | cut -d '.' -f1); echo "$TEMPLATE-mappings"; so-elasticsearch-query _component_template/$TEMPLATE-mappings -d@$i -XPUT 2>/dev/null; echo; done # Load SO-specific component templates cd ${ELASTICSEARCH_TEMPLATES}/component/so echo "Loading Security Onion component templates..." -for i in *; do TEMPLATE=$(echo $i | cut -d '.' -f1); echo "$TEMPLATE"; {{ ELASTICCURL }} -k ${ELASTICSEARCH_AUTH} -s -XPUT -L https://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_component_template/$TEMPLATE -H 'Content-Type: application/json' -d@$i 2>/dev/null; echo; done +for i in *; do TEMPLATE=$(echo $i | cut -d '.' -f1); echo "$TEMPLATE"; so-elasticsearch-query _component_template/$TEMPLATE -d@$i -XPUT 2>/dev/null; echo; done echo -if [ -d ${ELASTICSEARCH_TEMPLATES}/component/custom ]; then - # Load custom component templates - cd ${ELASTICSEARCH_TEMPLATES}/component/custom - - echo "Loading custom component templates..." - for i in *; do TEMPLATE=$(echo $i | cut -d '.' -f1); echo "$TEMPLATE"; curl -K /opt/so/conf/elasticsearch/curl.config -k ${ELASTICSEARCH_AUTH} -s -XPUT -L https://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_component_template/$TEMPLATE -H 'Content-Type: application/json' -d@$i 2>/dev/null; echo; done - echo -fi - # Load SO index templates cd ${ELASTICSEARCH_TEMPLATES}/index echo "Loading Security Onion index templates..." -for i in *; do TEMPLATE=$(echo $i | cut -d '-' -f2); echo "so-$TEMPLATE"; {{ ELASTICCURL }} -k ${ELASTICSEARCH_AUTH} -s -XPUT -L https://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_index_template/so-$TEMPLATE -H 'Content-Type: application/json' -d@$i 2>/dev/null; echo; done +for i in *; do TEMPLATE=$(echo $i | cut -d '-' -f2); echo "so-$TEMPLATE"; so-elasticsearch-query _index_template/so-$TEMPLATE -d@$i -XPUT 2>/dev/null; echo; done echo cd - >/dev/null From 42d6c3a956b18f4767c33938270df678579e6f62 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Tue, 15 Mar 2022 14:55:04 +0000 Subject: [PATCH 12/12] Replace Elastic connection check using ELASTICCURL with so-elasticsearch-query --- salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load b/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load index 1b6ac4685..93c1c6298 100755 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load @@ -30,7 +30,7 @@ echo -n "Waiting for ElasticSearch..." COUNT=0 ELASTICSEARCH_CONNECTED="no" while [[ "$COUNT" -le 240 ]]; do - {{ ELASTICCURL }} -k --output /dev/null --silent --head --fail -L https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT" + so-elasticsearch-query -k --output /dev/null --silent --head --fail if [ $? -eq 0 ]; then ELASTICSEARCH_CONNECTED="yes" echo "connected!"