From 4c5099d42980b26465ea231f5ab8ad2c549bfbc0 Mon Sep 17 00:00:00 2001 From: defensivedepth Date: Tue, 29 Oct 2024 10:27:54 -0400 Subject: [PATCH 01/19] Initial support for local lookup --- salt/elasticfleet/config.sls | 8 +++ .../grid-nodes_general/so-ip-mappings.json | 35 ++++++++++ .../so/9805_output_elastic_agent.conf.jinja | 67 ++++++++++++------- 3 files changed, 84 insertions(+), 26 deletions(-) create mode 100644 salt/elasticfleet/files/integrations/grid-nodes_general/so-ip-mappings.json diff --git a/salt/elasticfleet/config.sls b/salt/elasticfleet/config.sls index 71bc369c6..1dcc45896 100644 --- a/salt/elasticfleet/config.sls +++ b/salt/elasticfleet/config.sls @@ -63,6 +63,14 @@ eastatedir: - group: 939 - makedirs: True +custommappingsdir: + file.directory: + - name: /nsm/custom-mappings + - user: 947 + - group: 939 + - makedirs: True + + eapackageupgrade: file.managed: - name: /usr/sbin/so-elastic-fleet-package-upgrade diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/so-ip-mappings.json b/salt/elasticfleet/files/integrations/grid-nodes_general/so-ip-mappings.json new file mode 100644 index 000000000..fdcd36815 --- /dev/null +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/so-ip-mappings.json @@ -0,0 +1,35 @@ +{ + "package": { + "name": "log", + "version": "" + }, + "name": "so-ip-mappings", + "namespace": "so", + "description": "IP Description mappings", + "policy_id": "so-grid-nodes_general", + "vars": {}, + "inputs": { + "logs-logfile": { + "enabled": true, + "streams": { + "log.logs": { + "enabled": true, + "vars": { + "paths": [ + "/nsm/custom-mappings/ip-descriptions.csv" + ], + "data_stream.dataset": "hostnamemappings", + "tags": [ + "so-ip-mappings" + ], + "processors": "- decode_csv_fields:\n fields:\n message: decoded.csv\n separator: \",\"\n ignore_missing: false\n overwrite_keys: true\n trim_leading_space: true\n fail_on_error: true\n\n- extract_array:\n field: decoded.csv\n mappings:\n so.ip_address: '0'\n so.description: '1'\n\n- script:\n lang: javascript\n source: >\n function process(event) {\n var ip = event.Get('so.ip_address');\n var validIpRegex = /^((25[0-5]|2[0-4]\\d|1\\d{2}|[1-9]?\\d)\\.){3}(25[0-5]|2[0-4]\\d|1\\d{2}|[1-9]?\\d)$/\n if (!validIpRegex.test(ip)) {\n event.Cancel();\n }\n }\n- fingerprint:\n fields: [\"so.ip_address\"]\n target_field: \"@metadata._id\"\n", + "custom": "" + } + } + } + } + }, + "force": true +} + + diff --git a/salt/logstash/pipelines/config/so/9805_output_elastic_agent.conf.jinja b/salt/logstash/pipelines/config/so/9805_output_elastic_agent.conf.jinja index 3a86cd8be..be7ec6898 100644 --- a/salt/logstash/pipelines/config/so/9805_output_elastic_agent.conf.jinja +++ b/salt/logstash/pipelines/config/so/9805_output_elastic_agent.conf.jinja @@ -1,18 +1,45 @@ output { - if "elastic-agent" in [tags] { - if [metadata][pipeline] { - if [metadata][_id] { - elasticsearch { - hosts => "{{ GLOBALS.hostname }}" - ecs_compatibility => v8 - data_stream => true - user => "{{ ES_USER }}" - password => "{{ ES_PASS }}" - document_id => "%{[metadata][_id]}" - pipeline => "%{[metadata][pipeline]}" - silence_errors_in_log => ["version_conflict_engine_exception"] - ssl => true - ssl_certificate_verification => false + if "elastic-agent" in [tags] and "so-ip-mappings" in [tags] { + elasticsearch { + hosts => "{{ GLOBALS.hostname }}" + data_stream => false + user => "{{ ES_USER }}" + password => "{{ ES_PASS }}" + document_id => "%{[metadata][_id]}" + index => "so-ip-mappings" + silence_errors_in_log => ["version_conflict_engine_exception"] + ssl => true + ssl_certificate_verification => false + } + } + else { + if "elastic-agent" in [tags] { + if [metadata][pipeline] { + if [metadata][_id] { + elasticsearch { + hosts => "{{ GLOBALS.hostname }}" + ecs_compatibility => v8 + data_stream => true + user => "{{ ES_USER }}" + password => "{{ ES_PASS }}" + document_id => "%{[metadata][_id]}" + pipeline => "%{[metadata][pipeline]}" + silence_errors_in_log => ["version_conflict_engine_exception"] + ssl => true + ssl_certificate_verification => false + } + } + else { + elasticsearch { + hosts => "{{ GLOBALS.hostname }}" + ecs_compatibility => v8 + data_stream => true + user => "{{ ES_USER }}" + password => "{{ ES_PASS }}" + pipeline => "%{[metadata][pipeline]}" + ssl => true + ssl_certificate_verification => false + } } } else { @@ -22,22 +49,10 @@ output { data_stream => true user => "{{ ES_USER }}" password => "{{ ES_PASS }}" - pipeline => "%{[metadata][pipeline]}" ssl => true ssl_certificate_verification => false } } } - else { - elasticsearch { - hosts => "{{ GLOBALS.hostname }}" - ecs_compatibility => v8 - data_stream => true - user => "{{ ES_USER }}" - password => "{{ ES_PASS }}" - ssl => true - ssl_certificate_verification => false - } - } } } From 5406a263d57c90e10d4d6ee4f07b788ce20b64bd Mon Sep 17 00:00:00 2001 From: defensivedepth Date: Tue, 29 Oct 2024 19:42:06 -0400 Subject: [PATCH 02/19] Add local custom template --- salt/elasticfleet/config.sls | 9 +--- salt/manager/init.sls | 9 ++++ salt/soc/config.sls | 32 +++++++++++++ salt/soc/defaults.yaml | 12 +++++ ...tections_custom_repo_template_readme.jinja | 46 +++++++++++++++++++ 5 files changed, 100 insertions(+), 8 deletions(-) create mode 100644 salt/soc/files/soc/detections_custom_repo_template_readme.jinja diff --git a/salt/elasticfleet/config.sls b/salt/elasticfleet/config.sls index 1dcc45896..208fa2306 100644 --- a/salt/elasticfleet/config.sls +++ b/salt/elasticfleet/config.sls @@ -81,14 +81,7 @@ eapackageupgrade: - template: jinja {% if GLOBALS.role != "so-fleet" %} - -soresourcesrepoconfig: - git.config_set: - - name: safe.directory - - value: /nsm/securityonion-resources - - global: True - - user: socore - + {% if not GLOBALS.airgap %} soresourcesrepoclone: git.latest: diff --git a/salt/manager/init.sls b/salt/manager/init.sls index 96055df24..ca1296383 100644 --- a/salt/manager/init.sls +++ b/salt/manager/init.sls @@ -141,6 +141,15 @@ rules_dir: - group: socore - makedirs: True +git_config_set_safe_dirs: + git.config_set: + - name: safe.directory + - multivar: + - /nsm/rules/custom-local-repos/local-sigma + - /nsm/rules/custom-local-repos/local-yara + - /nsm/securityonion-resources + - /opt/so/conf/soc/ai_summary_repos/securityonion-resources + - global: True {% else %} {{sls}}_state_not_allowed: diff --git a/salt/soc/config.sls b/salt/soc/config.sls index 7607da5ff..52281db74 100644 --- a/salt/soc/config.sls +++ b/salt/soc/config.sls @@ -198,6 +198,38 @@ socsensoronirepos: - mode: 775 - makedirs: True +create_custom_local_yara_repo_template: + git.present: + - name: /nsm/rules/custom-local-repos/local-yara + - bare: False + - force: True + +add_readme_custom_local_yara_repo_template: + file.managed: + - name: /nsm/rules/custom-local-repos/local-yara/README + - source: salt://soc/files/soc/detections_custom_repo_template_readme.jinja + - user: 939 + - group: 939 + - template: jinja + - context: + repo_type: "yara" + +create_custom_local_sigma_repo_template: + git.present: + - name: /nsm/rules/custom-local-repos/local-sigma + - bare: False + - force: True + +add_readme_custom_local_sigma_repo_template: + file.managed: + - name: /nsm/rules/custom-local-repos/local-sigma/README + - source: salt://soc/files/soc/detections_custom_repo_template_readme.jinja + - user: 939 + - group: 939 + - template: jinja + - context: + repo_type: "sigma" + {% else %} {{sls}}_state_not_allowed: diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 6a9a1bfc6..fe4edb12b 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1342,11 +1342,17 @@ soc: license: Elastic-2.0 folder: sigma/stable community: true + - repo: file:///nsm/rules/custom-local-repos/local-sigma + license: Elastic-2.0 + community: false airgap: - repo: file:///nsm/rules/detect-sigma/repos/securityonion-resources license: Elastic-2.0 folder: sigma/stable community: true + - repo: file:///nsm/rules/custom-local-repos/local-sigma + license: Elastic-2.0 + community: false sigmaRulePackages: - core - emerging_threats_addon @@ -1412,10 +1418,16 @@ soc: - repo: https://github.com/Security-Onion-Solutions/securityonion-yara license: DRL community: true + - repo: file:///nsm/rules/custom-local-repos/local-yara + license: Elastic-2.0 + community: false airgap: - repo: file:///nsm/rules/detect-yara/repos/securityonion-yara license: DRL community: true + - repo: file:///nsm/rules/custom-local-repos/local-yara + license: Elastic-2.0 + community: false yaraRulesFolder: /opt/sensoroni/yara/rules stateFilePath: /opt/sensoroni/fingerprints/strelkaengine.state integrityCheckFrequencySeconds: 1200 diff --git a/salt/soc/files/soc/detections_custom_repo_template_readme.jinja b/salt/soc/files/soc/detections_custom_repo_template_readme.jinja new file mode 100644 index 000000000..1d391fec0 --- /dev/null +++ b/salt/soc/files/soc/detections_custom_repo_template_readme.jinja @@ -0,0 +1,46 @@ +{% if repo_type == 'yara' %} +# YARA Local Custom Rules Repository + +This folder has already been initialized as a git repo +and your Security Onion grid is configured to import any YARA rule files found here. + +Just add your rule file and commit it. + +For example: + +First, create the rule file; make sure to create the file with a .yar extension +`sudo vi my_custom_rule.yar` + +Next, use git to stage the new rule to be commited: +`sudo git add my_custom_rule.yar` + +Finally, commit it. +If this is your first time making changes to this repo, you will be asked to set some configuration. +`sudo git commit -m "Initial commit of my_custom_rule.yar"` + +The next time the Strelka / YARA engine syncs, the new rule should be imported +If there are errors, review the sync log to troubleshoot further. + +{% elif repo_type == 'sigma' %} +# Sigma Local Custom Rules Repository + +This folder has already been initialized as a git repo +and your Security Onion grid is configured to import any Sigma rule files found here. + +Just add your rule file and commit it. + +For example: + +First, create the rule file; make sure to create the file with a .yaml|.yml extension +`sudo vi my_custom_rule.yml` + +Next, use git to stage the new rule to be commited: +`sudo git add my_custom_rule.yml` + +Finally, commit it. +If this is your first time making changes to this repo, you will be asked to set some configuration. +`sudo git commit -m "Initial commit of my_custom_rule.yml"` + +The next time the Elastalert / Sigma engine syncs, the new rule should be imported +If there are errors, review the sync log to troubleshoot further. +{% endif %} \ No newline at end of file From 36fc3bbd6dd0df4d9ba4110f6269e3605ec8e347 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Wed, 30 Oct 2024 10:24:11 -0400 Subject: [PATCH 03/19] add so-ip-mappings index Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/elasticsearch/defaults.yaml | 29 +++++++++++++++++++ .../component/so/so-ip-mappings.json | 22 ++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 salt/elasticsearch/templates/component/so/so-ip-mappings.json diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index f0178728e..c8684e775 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -599,6 +599,35 @@ elasticsearch: set_priority: priority: 50 min_age: 30d + so-ip-mappings: + index_sorting: false + index_template: + composed_of: + - so-ip-mappings + ignore_missing_component_templates: [] + index_patterns: + - so-ip* + priority: 500 + template: + mappings: + date_detection: false + dynamic_templates: + - strings_as_keyword: + mapping: + ignore_above: 1024 + type: keyword + match_mapping_type: string + settings: + index: + mapping: + total_fields: + limit: 1500 + number_of_replicas: 0 + number_of_shards: 1 + refresh_interval: 30s + sort: + field: '@timestamp' + order: desc so-items: index_sorting: false index_template: diff --git a/salt/elasticsearch/templates/component/so/so-ip-mappings.json b/salt/elasticsearch/templates/component/so/so-ip-mappings.json new file mode 100644 index 000000000..ab80e365c --- /dev/null +++ b/salt/elasticsearch/templates/component/so/so-ip-mappings.json @@ -0,0 +1,22 @@ +{ + "_meta": { + "documentation": "https://www.elastic.co/guide/en/ecs/current/ecs-network.html", + "ecs_version": "1.12.2" + }, + "template": { + "mappings": { + "properties": { + "so": { + "properties": { + "ip_address": { + "type": "ip" + }, + "description": { + "type": "text" + } + } + } + } + } + } +} \ No newline at end of file From c509dab5f18d0c2c807197b88f7d7ca0dbdc822c Mon Sep 17 00:00:00 2001 From: defensivedepth Date: Wed, 30 Oct 2024 11:03:14 -0400 Subject: [PATCH 04/19] Use socore user --- salt/manager/init.sls | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/manager/init.sls b/salt/manager/init.sls index ca1296383..a3fb44e97 100644 --- a/salt/manager/init.sls +++ b/salt/manager/init.sls @@ -144,12 +144,13 @@ rules_dir: git_config_set_safe_dirs: git.config_set: - name: safe.directory + - global: True + - user: socore - multivar: - /nsm/rules/custom-local-repos/local-sigma - /nsm/rules/custom-local-repos/local-yara - /nsm/securityonion-resources - /opt/so/conf/soc/ai_summary_repos/securityonion-resources - - global: True {% else %} {{sls}}_state_not_allowed: From 6b468eaed3ec886442ed1cfcf496ff7c2dc4e489 Mon Sep 17 00:00:00 2001 From: defensivedepth Date: Wed, 30 Oct 2024 16:52:44 -0400 Subject: [PATCH 05/19] rm eaintegration state file --- salt/manager/tools/sbin/soup | 3 +++ 1 file changed, 3 insertions(+) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 3fda54fb9..bd2db98d7 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -719,6 +719,9 @@ up_to_2.4.120() { mkdir /opt/so/saltstack/local/pillar/versionlock touch /opt/so/saltstack/local/pillar/versionlock/adv_versionlock.sls /opt/so/saltstack/local/pillar/versionlock/soc_versionlock.sls + # New Grid Integration added this release + rm -f /opt/so/state/eaintegrations.txt + INSTALLEDVERSION=2.4.120 } From 7896f951f3f7f267b7db65febf83b255280f8635 Mon Sep 17 00:00:00 2001 From: defensivedepth Date: Thu, 31 Oct 2024 10:24:58 -0400 Subject: [PATCH 06/19] timestamp fix --- salt/elasticsearch/templates/component/so/so-ip-mappings.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/salt/elasticsearch/templates/component/so/so-ip-mappings.json b/salt/elasticsearch/templates/component/so/so-ip-mappings.json index ab80e365c..a61eae5fd 100644 --- a/salt/elasticsearch/templates/component/so/so-ip-mappings.json +++ b/salt/elasticsearch/templates/component/so/so-ip-mappings.json @@ -6,6 +6,9 @@ "template": { "mappings": { "properties": { + "@timestamp": { + "type": "date" + }, "so": { "properties": { "ip_address": { From 083c678400f1905cb0abe25dee0ea22551f75e83 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 4 Nov 2024 09:46:26 -0500 Subject: [PATCH 07/19] new salt repo --- setup/so-functions | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index e3c8a5615..cc55ec21d 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1843,9 +1843,9 @@ repo_sync_local() { fi dnf install -y yum-utils device-mapper-persistent-data lvm2 curl -fsSL https://repo.securityonion.net/file/so-repo/prod/2.4/so/so.repo | tee /etc/yum.repos.d/so.repo - rpm --import https://repo.saltproject.io/salt/py3/redhat/9/x86_64/SALT-PROJECT-GPG-PUBKEY-2023.pub + rpm --import https://packages.broadcom.com/artifactory/api/security/keypair/SaltProjectKey/public dnf config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo - curl -fsSL "https://repo.saltproject.io/salt/py3/redhat/9/x86_64/minor/$SALTVERSION.repo" | tee /etc/yum.repos.d/salt.repo + curl -fsSL "https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" | tee /etc/yum.repos.d/salt.repo dnf repolist curl --retry 5 --retry-delay 60 -A "netinstall/$SOVERSION/$OS/$(uname -r)/1" https://sigs.securityonion.net/checkup --output /tmp/install else @@ -1878,24 +1878,19 @@ saltify() { logCmd "mkdir -vp /etc/apt/keyrings" logCmd "wget -q --inet4-only -O /etc/apt/keyrings/docker.pub https://download.docker.com/linux/ubuntu/gpg" + # Download public key + logCmd "curl -fsSL -o /etc/apt/keyrings/salt-archive-keyring-2023.pgp https://packages.broadcom.com/artifactory/api/security/keypair/SaltProjectKey/public" + # Create apt repo target configuration + echo "deb [signed-by=/etc/apt/keyrings/salt-archive-keyring-2023.pgp arch=amd64] https://packages.broadcom.com/artifactory/saltproject-deb/ stable main" | sudo tee /etc/apt/sources.list.d/salt.list + if [[ $is_ubuntu ]]; then - - # Add Salt Repo - logCmd "curl -fsSL -o /etc/apt/keyrings/salt-archive-keyring-2023.gpg https://repo.saltproject.io/salt/py3/ubuntu/$UBVER/amd64/minor/$SALTVERSION/SALT-PROJECT-GPG-PUBKEY-2023.gpg" - echo "deb [signed-by=/etc/apt/keyrings/salt-archive-keyring-2023.gpg] https://repo.saltproject.io/salt/py3/ubuntu/$UBVER/amd64/minor/$SALTVERSION/ $OSVER main" | sudo tee /etc/apt/sources.list.d/salt.list - # Add Docker Repo add-apt-repository -y "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" else - # Add Salt Repo *NOTE* You have to use debian 11 since it isn't out for 12 - logCmd "curl -fsSL -o /etc/apt/keyrings/salt-archive-keyring-2023.gpg https://repo.saltproject.io/salt/py3/debian/11/amd64/minor/$SALTVERSION/SALT-PROJECT-GPG-PUBKEY-2023.gpg" - echo "deb [signed-by=/etc/apt/keyrings/salt-archive-keyring-2023.gpg] https://repo.saltproject.io/salt/py3/debian/11/amd64/minor/$SALTVERSION/ bullseye main" | sudo tee /etc/apt/sources.list.d/salt.list - # Add Docker Repo curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg echo "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian $OSVER stable" > /etc/apt/sources.list.d/docker.list - fi logCmd "apt-key add /etc/apt/keyrings/salt-archive-keyring-2023.gpg" From 6e14f7b6267d7419c9ac3807a1817dcde1280918 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 4 Nov 2024 11:14:00 -0500 Subject: [PATCH 08/19] fix pub key name --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index cc55ec21d..c6aadef3d 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1893,7 +1893,7 @@ saltify() { echo "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian $OSVER stable" > /etc/apt/sources.list.d/docker.list fi - logCmd "apt-key add /etc/apt/keyrings/salt-archive-keyring-2023.gpg" + logCmd "apt-key add /etc/apt/keyrings/salt-archive-keyring-2023.pgp" #logCmd "apt-key add /opt/so/gpg/SALTSTACK-GPG-KEY.pub" logCmd "apt-key add /etc/apt/keyrings/docker.pub" From 69dd35c30a49587f08261c40f2b56458a7dbc96b Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Mon, 4 Nov 2024 14:31:53 -0700 Subject: [PATCH 09/19] Add Option for Ignoring Ranges of SIDs in Suricata Integrity Check --- salt/soc/defaults.yaml | 2 ++ salt/soc/soc_soc.yaml | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 6a9a1bfc6..068a9c9b7 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1435,6 +1435,8 @@ soc: rulesFingerprintFile: /opt/sensoroni/fingerprints/emerging-all.fingerprint stateFilePath: /opt/sensoroni/fingerprints/suricataengine.state integrityCheckFrequencySeconds: 1200 + ignoredSidRanges: + - '1100000-1199999' client: enableReverseLookup: false docsUrl: /docs/ diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index 14296dade..fd3295daf 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -390,6 +390,11 @@ soc: advanced: True forcedType: "[]{}" helpLink: suricata.html + ignoredSidRanges: + description: 'List of Suricata SID ranges to ignore during the Integrity Check. This is useful for ignoring specific rules not governed by the UI.' + global: True + advanced: True + forcedType: "[]string" client: enableReverseLookup: description: Set to true to enable reverse DNS lookups for IP addresses in the SOC UI. From 5e48ccafce5ab3b4a04d6e059b311b8efac975b4 Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Tue, 5 Nov 2024 11:11:34 -0700 Subject: [PATCH 10/19] Update Default Value --- salt/soc/defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 068a9c9b7..73446abd7 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1436,7 +1436,7 @@ soc: stateFilePath: /opt/sensoroni/fingerprints/suricataengine.state integrityCheckFrequencySeconds: 1200 ignoredSidRanges: - - '1100000-1199999' + - '1100000-1101000' client: enableReverseLookup: false docsUrl: /docs/ From 25d55feeefd8f6955271daac729b94be1d2e4d4a Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Tue, 5 Nov 2024 11:41:14 -0700 Subject: [PATCH 11/19] More Detailed Description --- salt/soc/soc_soc.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index fd3295daf..3bd1774bc 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -391,7 +391,7 @@ soc: forcedType: "[]{}" helpLink: suricata.html ignoredSidRanges: - description: 'List of Suricata SID ranges to ignore during the Integrity Check. This is useful for ignoring specific rules not governed by the UI.' + description: 'List of Suricata SID ranges to ignore during the Integrity Check. This is useful for ignoring specific rules not governed by the UI. Each line should contain 1 range in the format "1100000-1200000". The ranges are treated as inclusive.' global: True advanced: True forcedType: "[]string" From 52a144c0521a3a4f5b7e476f60251b66f2c0a332 Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Tue, 5 Nov 2024 12:11:17 -0700 Subject: [PATCH 12/19] Added Help Link to Annotation for IgnoredSidRanges --- salt/soc/soc_soc.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index 3bd1774bc..c27228ab6 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -395,6 +395,7 @@ soc: global: True advanced: True forcedType: "[]string" + helpLink: detections.html#rule-engine-status client: enableReverseLookup: description: Set to true to enable reverse DNS lookups for IP addresses in the SOC UI. From 039d5c22ac8212c01bdd68a5e5afbcccb4b532a9 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Wed, 6 Nov 2024 14:35:41 -0600 Subject: [PATCH 13/19] fix: crowdstrike integration Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/elasticsearch/defaults.yaml | 120 +++++++++++++++--- salt/elasticsearch/soc_elasticsearch.yaml | 2 + .../logs-crowdstrike.alert@custom.json | 36 ++++++ .../logs-crowdstrike.host@custom.json | 36 ++++++ 4 files changed, 176 insertions(+), 18 deletions(-) create mode 100644 salt/elasticsearch/templates/component/elastic-agent/logs-crowdstrike.alert@custom.json create mode 100644 salt/elasticsearch/templates/component/elastic-agent/logs-crowdstrike.host@custom.json diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index c8684e775..e3259ecd5 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -3499,28 +3499,70 @@ elasticsearch: set_priority: priority: 50 min_age: 30d - so-logs-crowdstrike_x_falcon: - index_sorting: false + so-logs-crowdstrike_x_alert: + index_sorting: False index_template: + index_patterns: + - logs-crowdstrike.alert-* + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - logs-crowdstrike.alert@package + - logs-crowdstrike.alert@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + ignore_missing_component_templates: + - logs-crowdstrike.alert@custom + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-crowdstrike_x_falcon: + index_sorting: False + index_template: + index_patterns: + - logs-crowdstrike.falcon-* + template: + settings: + index: + number_of_replicas: 0 composed_of: - logs-crowdstrike.falcon@package - logs-crowdstrike.falcon@custom - so-fleet_globals-1 - so-fleet_agent_id_verification-1 + priority: 501 data_stream: - allow_custom_routing: false hidden: false + allow_custom_routing: false ignore_missing_component_templates: - logs-crowdstrike.falcon@custom - index_patterns: - - logs-crowdstrike.falcon-* - priority: 501 - template: - settings: - index: - lifecycle: - name: so-logs-crowdstrike.falcon-logs - number_of_replicas: 0 policy: phases: cold: @@ -3546,27 +3588,69 @@ elasticsearch: priority: 50 min_age: 30d so-logs-crowdstrike_x_fdr: - index_sorting: false + index_sorting: False index_template: + index_patterns: + - logs-crowdstrike.fdr-* + template: + settings: + index: + number_of_replicas: composed_of: - logs-crowdstrike.fdr@package - logs-crowdstrike.fdr@custom - so-fleet_globals-1 - so-fleet_agent_id_verification-1 + priority: 501 data_stream: - allow_custom_routing: false hidden: false + allow_custom_routing: false ignore_missing_component_templates: - logs-crowdstrike.fdr@custom + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-crowdstrike_x_host: + index_sorting: False + index_template: index_patterns: - - logs-crowdstrike.fdr-* - priority: 501 + - logs-crowdstrike.host-* template: settings: index: - lifecycle: - name: so-logs-crowdstrike.fdr-logs number_of_replicas: 0 + composed_of: + - logs-crowdstrike.host@package + - logs-crowdstrike.host@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + ignore_missing_component_templates: + - logs-crowdstrike.host@custom policy: phases: cold: diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index 266372708..e26d1d705 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -396,8 +396,10 @@ elasticsearch: so-logs-citrix_waf_x_log: *indexSettings so-logs-cloudflare_x_audit: *indexSettings so-logs-cloudflare_x_logpull: *indexSettings + so-logs-crowdstrike_x_alert: *indexSettings so-logs-crowdstrike_x_falcon: *indexSettings so-logs-crowdstrike_x_fdr: *indexSettings + so-logs-crowdstrike_x_host: *indexSettings so-logs-darktrace_x_ai_analyst_alert: *indexSettings so-logs-darktrace_x_model_breach_alert: *indexSettings so-logs-darktrace_x_system_status_alert: *indexSettings diff --git a/salt/elasticsearch/templates/component/elastic-agent/logs-crowdstrike.alert@custom.json b/salt/elasticsearch/templates/component/elastic-agent/logs-crowdstrike.alert@custom.json new file mode 100644 index 000000000..17319ab9f --- /dev/null +++ b/salt/elasticsearch/templates/component/elastic-agent/logs-crowdstrike.alert@custom.json @@ -0,0 +1,36 @@ +{ + "template": { + "mappings": { + "properties": { + "host": { + "properties":{ + "ip": { + "type": "ip" + } + } + }, + "related": { + "properties":{ + "ip": { + "type": "ip" + } + } + }, + "destination": { + "properties":{ + "ip": { + "type": "ip" + } + } + }, + "source": { + "properties":{ + "ip": { + "type": "ip" + } + } + } + } + } + } +} diff --git a/salt/elasticsearch/templates/component/elastic-agent/logs-crowdstrike.host@custom.json b/salt/elasticsearch/templates/component/elastic-agent/logs-crowdstrike.host@custom.json new file mode 100644 index 000000000..17319ab9f --- /dev/null +++ b/salt/elasticsearch/templates/component/elastic-agent/logs-crowdstrike.host@custom.json @@ -0,0 +1,36 @@ +{ + "template": { + "mappings": { + "properties": { + "host": { + "properties":{ + "ip": { + "type": "ip" + } + } + }, + "related": { + "properties":{ + "ip": { + "type": "ip" + } + } + }, + "destination": { + "properties":{ + "ip": { + "type": "ip" + } + } + }, + "source": { + "properties":{ + "ip": { + "type": "ip" + } + } + } + } + } + } +} From 80b82b0bd62b61739b0c689aa2e75967f35fc4af Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Wed, 6 Nov 2024 15:24:13 -0600 Subject: [PATCH 14/19] missing replica 0 Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/elasticsearch/defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index e3259ecd5..133c333e1 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -3595,7 +3595,7 @@ elasticsearch: template: settings: index: - number_of_replicas: + number_of_replicas: 0 composed_of: - logs-crowdstrike.fdr@package - logs-crowdstrike.fdr@custom From f5bd8ab58556c0f9bafa26717d83de33f6dd2862 Mon Sep 17 00:00:00 2001 From: defensivedepth Date: Thu, 7 Nov 2024 15:33:47 -0500 Subject: [PATCH 15/19] Rewrite docs --- ...tections_custom_repo_template_readme.jinja | 76 +++++++++++++++---- 1 file changed, 62 insertions(+), 14 deletions(-) diff --git a/salt/soc/files/soc/detections_custom_repo_template_readme.jinja b/salt/soc/files/soc/detections_custom_repo_template_readme.jinja index 1d391fec0..228a467bf 100644 --- a/salt/soc/files/soc/detections_custom_repo_template_readme.jinja +++ b/salt/soc/files/soc/detections_custom_repo_template_readme.jinja @@ -8,15 +8,39 @@ Just add your rule file and commit it. For example: -First, create the rule file; make sure to create the file with a .yar extension -`sudo vi my_custom_rule.yar` +** Note: If this is your first time making changes to this repo, you may run into the following error: -Next, use git to stage the new rule to be commited: -`sudo git add my_custom_rule.yar` +fatal: detected dubious ownership in repository at '/nsm/rules/custom-local-repos/local-yara' +To add an exception for this directory, call: + git config --global --add safe.directory /nsm/rules/custom-local-repos/local-yara -Finally, commit it. -If this is your first time making changes to this repo, you will be asked to set some configuration. -`sudo git commit -m "Initial commit of my_custom_rule.yar"` +This means that the user you are running commands as does not match the user that is used for this git repo (socore). +You will need to make sure your rule files are accessible to the socore user, so either su to socore +or add the exception and then chown the rule files later. + +Also, you will be asked to set some configuration: +``` +Author identity unknown +*** Please tell me who you are. +Run + git config --global user.email "you@example.com" + git config --global user.name "Your Name" +to set your account's default identity. +Omit --global to set the identity only in this repository. +``` + +Run these commands, ommitting the `--global`. + +With that out of the way: + +First, create the rule file with a .yar extension: +`vi my_custom_rule.yar` + +Next, use git to stage the new rule to be committed: +`git add my_custom_rule.yar` + +Finally, commit it: +`git commit -m "Initial commit of my_custom_rule.yar"` The next time the Strelka / YARA engine syncs, the new rule should be imported If there are errors, review the sync log to troubleshoot further. @@ -31,15 +55,39 @@ Just add your rule file and commit it. For example: -First, create the rule file; make sure to create the file with a .yaml|.yml extension -`sudo vi my_custom_rule.yml` +** Note: If this is your first time making changes to this repo, you may run into the following error: -Next, use git to stage the new rule to be commited: -`sudo git add my_custom_rule.yml` +fatal: detected dubious ownership in repository at '/nsm/rules/custom-local-repos/local-sigma' +To add an exception for this directory, call: + git config --global --add safe.directory /nsm/rules/custom-local-repos/local-sigma -Finally, commit it. -If this is your first time making changes to this repo, you will be asked to set some configuration. -`sudo git commit -m "Initial commit of my_custom_rule.yml"` +This means that the user you are running commands as does not match the user that is used for this git repo (socore). +You will need to make sure your rule files are accessible to the socore user, so either su to socore +or add the exception and then chown the rule files later. + +Also, you will be asked to set some configuration: +``` +Author identity unknown +*** Please tell me who you are. +Run + git config --global user.email "you@example.com" + git config --global user.name "Your Name" +to set your account's default identity. +Omit --global to set the identity only in this repository. +``` + +Run these commands, ommitting the `--global`. + +With that out of the way: + +First, create the rule file with a .yml or .yaml extension: +`vi my_custom_rule.yml` + +Next, use git to stage the new rule to be committed: +`git add my_custom_rule.yml` + +Finally, commit it: +`git commit -m "Initial commit of my_custom_rule.yml"` The next time the Elastalert / Sigma engine syncs, the new rule should be imported If there are errors, review the sync log to troubleshoot further. From 8334fd9c46d80ef12cc127b9a8d5c14eba04e0ac Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Wed, 6 Nov 2024 10:52:34 -0700 Subject: [PATCH 16/19] Source Dates --- .../templates/component/so/detection-mappings.json | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/salt/elasticsearch/templates/component/so/detection-mappings.json b/salt/elasticsearch/templates/component/so/detection-mappings.json index 2e405912d..4dd5b45e7 100644 --- a/salt/elasticsearch/templates/component/so/detection-mappings.json +++ b/salt/elasticsearch/templates/component/so/detection-mappings.json @@ -64,7 +64,7 @@ }, "tags": { "ignore_above": 1024, - "type": "keyword" + "type": "keyword" }, "ruleset": { "ignore_above": 1024, @@ -82,6 +82,12 @@ "ignore_above": 1024, "type": "keyword" }, + "sourceCreated": { + "type": "date" + }, + "sourceUpdated": { + "type": "date" + }, "overrides": { "properties": { "type": { From 8b70aa9f0eedc3e05372362eae5bce1301628495 Mon Sep 17 00:00:00 2001 From: defensivedepth Date: Fri, 8 Nov 2024 09:19:41 -0500 Subject: [PATCH 17/19] Fix socore permissions --- salt/soc/config.sls | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/salt/soc/config.sls b/salt/soc/config.sls index 52281db74..c153ad7a6 100644 --- a/salt/soc/config.sls +++ b/salt/soc/config.sls @@ -198,11 +198,22 @@ socsensoronirepos: - mode: 775 - makedirs: True +make-repo-dir-yara: + file.directory: + - name: /nsm/rules/custom-local-repos/local-yara + - user: socore + - group: socore + - makedirs: True + - recurse: + - user + - group + create_custom_local_yara_repo_template: git.present: - name: /nsm/rules/custom-local-repos/local-yara - bare: False - force: True + - user: socore add_readme_custom_local_yara_repo_template: file.managed: @@ -214,11 +225,22 @@ add_readme_custom_local_yara_repo_template: - context: repo_type: "yara" +make-repo-dir-sigma: + file.directory: + - name: /nsm/rules/custom-local-repos/local-sigma + - user: socore + - group: socore + - makedirs: True + - recurse: + - user + - group + create_custom_local_sigma_repo_template: git.present: - name: /nsm/rules/custom-local-repos/local-sigma - bare: False - force: True + - user: socore add_readme_custom_local_sigma_repo_template: file.managed: From dcbb0e48d4684fc676d72a27e1744a84737d16ae Mon Sep 17 00:00:00 2001 From: defensivedepth Date: Fri, 8 Nov 2024 14:34:29 -0500 Subject: [PATCH 18/19] make sure its owned by socore --- salt/soc/config.sls | 29 +++++++++-------------------- 1 file changed, 9 insertions(+), 20 deletions(-) diff --git a/salt/soc/config.sls b/salt/soc/config.sls index c153ad7a6..89627d659 100644 --- a/salt/soc/config.sls +++ b/salt/soc/config.sls @@ -198,22 +198,12 @@ socsensoronirepos: - mode: 775 - makedirs: True -make-repo-dir-yara: - file.directory: - - name: /nsm/rules/custom-local-repos/local-yara - - user: socore - - group: socore - - makedirs: True - - recurse: - - user - - group create_custom_local_yara_repo_template: git.present: - name: /nsm/rules/custom-local-repos/local-yara - bare: False - force: True - - user: socore add_readme_custom_local_yara_repo_template: file.managed: @@ -225,22 +215,12 @@ add_readme_custom_local_yara_repo_template: - context: repo_type: "yara" -make-repo-dir-sigma: - file.directory: - - name: /nsm/rules/custom-local-repos/local-sigma - - user: socore - - group: socore - - makedirs: True - - recurse: - - user - - group create_custom_local_sigma_repo_template: git.present: - name: /nsm/rules/custom-local-repos/local-sigma - bare: False - force: True - - user: socore add_readme_custom_local_sigma_repo_template: file.managed: @@ -251,6 +231,15 @@ add_readme_custom_local_sigma_repo_template: - template: jinja - context: repo_type: "sigma" + +socore_own_custom_repos: + file.directory: + - name: /nsm/rules/custom-local-repos/ + - user: socore + - group: socore + - recurse: + - user + - group {% else %} From ee4405e75ea27035e43304932ca3d5c2d3cb38b0 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 8 Nov 2024 16:13:44 -0500 Subject: [PATCH 19/19] only enable repo sync cron if OEL --- salt/manager/init.sls | 5 +---- salt/manager/map.jinja | 6 +++++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/salt/manager/init.sls b/salt/manager/init.sls index 96055df24..07c39b371 100644 --- a/salt/manager/init.sls +++ b/salt/manager/init.sls @@ -6,10 +6,7 @@ {% from 'allowed_states.map.jinja' import allowed_states %} {% if sls in allowed_states %} {% from 'vars/globals.map.jinja' import GLOBALS %} -{% from 'strelka/map.jinja' import STRELKAMERGED %} -{% import_yaml 'manager/defaults.yaml' as MANAGERDEFAULTS %} -{% set MANAGERMERGED = salt['pillar.get']('manager', MANAGERDEFAULTS.manager, merge=true) %} -{% from 'strelka/map.jinja' import STRELKAMERGED %} +{% from 'manager/map.jinja' import MANAGERMERGED %} include: - salt.minion diff --git a/salt/manager/map.jinja b/salt/manager/map.jinja index 1ab9c12c3..4ea04a1cf 100644 --- a/salt/manager/map.jinja +++ b/salt/manager/map.jinja @@ -4,4 +4,8 @@ Elastic License 2.0. #} {% import_yaml 'manager/defaults.yaml' as MANAGERDEFAULTS %} -{% set MANAGERMERGED = salt['pillar.get']('manager', MANAGERDEFAULTS.manager, merge=True) %} \ No newline at end of file +{% set MANAGERMERGED = salt['pillar.get']('manager', MANAGERDEFAULTS.manager, merge=True) %} + +{% if grains.os != 'OEL' %} +{% do MANAGERMERGED.reposync.update({'enabled': False}) %} +{% endif %}