From 0fa4d92f8ff8beb7d8e597bf357ab3a578c04c51 Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Thu, 19 Oct 2023 15:49:56 -0600 Subject: [PATCH 01/17] socsigmarepo Need write permissions on the /opt/so/rules dir so I can clone the sigma repo there. --- salt/soc/config.sls | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/salt/soc/config.sls b/salt/soc/config.sls index 902d82ec7..95135566b 100644 --- a/salt/soc/config.sls +++ b/salt/soc/config.sls @@ -114,6 +114,13 @@ socuploaddir: - group: 939 - makedirs: True +socsigmarepo: + file.directory: + - name: /opt/so/rules + - user: 939 + - group: 939 + - mode: 775 + {% else %} {{sls}}_state_not_allowed: From 858166bcae280be03c5b416c912fd8c4a7ac8d61 Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Tue, 30 Jan 2024 15:43:51 -0700 Subject: [PATCH 02/17] WIP: Detections Changes Removed some strelka/yara rules from salt. Removed yara scripts for downloading and updating rules. This will be managed by SOC. Added a new compile_yara.py script. Added the strelka repos folder. --- salt/manager/init.sls | 53 +------------------ .../manager/tools/sbin_jinja/so-yara-download | 51 ------------------ salt/manager/tools/sbin_jinja/so-yara-update | 41 -------------- salt/soc/files/bin/compile_yara.py | 14 +++++ salt/strelka/backend/config.sls | 10 ---- salt/strelka/config.sls | 9 +++- 6 files changed, 24 insertions(+), 154 deletions(-) delete mode 100644 salt/manager/tools/sbin_jinja/so-yara-download delete mode 100755 salt/manager/tools/sbin_jinja/so-yara-update create mode 100644 salt/soc/files/bin/compile_yara.py diff --git a/salt/manager/init.sls b/salt/manager/init.sls index 23ef189b5..51590a6ec 100644 --- a/salt/manager/init.sls +++ b/salt/manager/init.sls @@ -1,5 +1,5 @@ # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. @@ -61,7 +61,7 @@ manager_sbin: - user: 939 - group: 939 - file_mode: 755 - - exclude_pat: + - exclude_pat: - "*_test.py" yara_update_scripts: @@ -103,55 +103,6 @@ rules_dir: - group: socore - makedirs: True -{% if STRELKAMERGED.rules.enabled %} - -strelkarepos: - file.managed: - - name: /opt/so/conf/strelka/repos.txt - - source: salt://strelka/rules/repos.txt.jinja - - template: jinja - - defaults: - STRELKAREPOS: {{ STRELKAMERGED.rules.repos }} - - makedirs: True - -strelka-yara-update: - {% if MANAGERMERGED.reposync.enabled and not GLOBALS.airgap %} - cron.present: - {% else %} - cron.absent: - {% endif %} - - user: socore - - name: '/usr/sbin/so-yara-update >> /opt/so/log/yarasync/yara-update.log 2>&1' - - identifier: strelka-yara-update - - hour: '7' - - minute: '1' - -strelka-yara-download: - {% if MANAGERMERGED.reposync.enabled and not GLOBALS.airgap %} - cron.present: - {% else %} - cron.absent: - {% endif %} - - user: socore - - name: '/usr/sbin/so-yara-download >> /opt/so/log/yarasync/yara-download.log 2>&1' - - identifier: strelka-yara-download - - hour: '7' - - minute: '1' - -{% if not GLOBALS.airgap %} -update_yara_rules: - cmd.run: - - name: /usr/sbin/so-yara-update - - onchanges: - - file: yara_update_scripts - -download_yara_rules: - cmd.run: - - name: /usr/sbin/so-yara-download - - onchanges: - - file: yara_update_scripts -{% endif %} -{% endif %} {% else %} {{sls}}_state_not_allowed: diff --git a/salt/manager/tools/sbin_jinja/so-yara-download b/salt/manager/tools/sbin_jinja/so-yara-download deleted file mode 100644 index aa9576253..000000000 --- a/salt/manager/tools/sbin_jinja/so-yara-download +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/bash -NOROOT=1 -. /usr/sbin/so-common - -{%- set proxy = salt['pillar.get']('manager:proxy') %} -{%- set noproxy = salt['pillar.get']('manager:no_proxy', '') %} - -# Download the rules from the internet -{%- if proxy %} -export http_proxy={{ proxy }} -export https_proxy={{ proxy }} -export no_proxy="{{ noproxy }}" -{%- endif %} - -repos="/opt/so/conf/strelka/repos.txt" -output_dir=/nsm/rules/yara -gh_status=$(curl -s -o /dev/null -w "%{http_code}" https://github.com) -clone_dir="/tmp" -if [ "$gh_status" == "200" ] || [ "$gh_status" == "301" ]; then - - while IFS= read -r repo; do - if ! $(echo "$repo" | grep -qE '^#'); then - # Remove old repo if existing bc of previous error condition or unexpected disruption - repo_name=`echo $repo | awk -F '/' '{print $NF}'` - [ -d $output_dir/$repo_name ] && rm -rf $output_dir/$repo_name - - # Clone repo and make appropriate directories for rules - git clone $repo $clone_dir/$repo_name - echo "Analyzing rules from $clone_dir/$repo_name..." - mkdir -p $output_dir/$repo_name - # Ensure a copy of the license is available for the rules - [ -f $clone_dir/$repo_name/LICENSE ] && cp $clone_dir/$repo_name/LICENSE $output_dir/$repo_name - - # Copy over rules - for i in $(find $clone_dir/$repo_name -name "*.yar*"); do - rule_name=$(echo $i | awk -F '/' '{print $NF}') - cp $i $output_dir/$repo_name - done - rm -rf $clone_dir/$repo_name - fi - done < $repos - - echo "Done!" - -/usr/sbin/so-yara-update - -else - echo "Server returned $gh_status status code." - echo "No connectivity to Github...exiting..." - exit 1 -fi diff --git a/salt/manager/tools/sbin_jinja/so-yara-update b/salt/manager/tools/sbin_jinja/so-yara-update deleted file mode 100755 index 07c940f47..000000000 --- a/salt/manager/tools/sbin_jinja/so-yara-update +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -NOROOT=1 -. /usr/sbin/so-common - -echo "Starting to check for yara rule updates at $(date)..." - -newcounter=0 -excludedcounter=0 -excluded_rules=({{ EXCLUDEDRULES | join(' ') }}) - -# Pull down the SO Rules -SORULEDIR=/nsm/rules/yara -OUTPUTDIR=/opt/so/saltstack/local/salt/strelka/rules - -mkdir -p $OUTPUTDIR -# remove all rules prior to copy so we can clear out old rules -rm -f $OUTPUTDIR/* - -for i in $(find $SORULEDIR -name "*.yar" -o -name "*.yara"); do - rule_name=$(echo $i | awk -F '/' '{print $NF}') - if [[ ! "${excluded_rules[*]}" =~ ${rule_name} ]]; then - echo "Adding rule: $rule_name..." - cp $i $OUTPUTDIR/$rule_name - ((newcounter++)) - else - echo "Excluding rule: $rule_name..." - ((excludedcounter++)) - fi -done - -if [ "$newcounter" -gt 0 ] || [ "$excludedcounter" -gt 0 ];then - echo "$newcounter rules added." - echo "$excludedcounter rule(s) excluded." -fi - -echo "Finished rule updates at $(date)..." diff --git a/salt/soc/files/bin/compile_yara.py b/salt/soc/files/bin/compile_yara.py new file mode 100644 index 000000000..43c8b1a09 --- /dev/null +++ b/salt/soc/files/bin/compile_yara.py @@ -0,0 +1,14 @@ +import os +import yara +import glob +import sys + +def compile_yara_rules(rules_dir: str) -> None: + compiled_rules_path: str = os.path.join(rules_dir, "rules.yar.compiled") + rule_files: list[str] = glob.glob(os.path.join(rules_dir, '**/*.yar'), recursive=True) + + if rule_files: + rules: yara.Rules = yara.compile(filepaths={os.path.basename(f): f for f in rule_files}) + rules.save(compiled_rules_path) + +compile_yara_rules(sys.argv[1]) diff --git a/salt/strelka/backend/config.sls b/salt/strelka/backend/config.sls index d51debb1b..b39e06ac8 100644 --- a/salt/strelka/backend/config.sls +++ b/salt/strelka/backend/config.sls @@ -50,16 +50,6 @@ backend_taste: - user: 939 - group: 939 -{% if STRELKAMERGED.rules.enabled %} -strelkarules: - file.recurse: - - name: /opt/so/conf/strelka/rules - - source: salt://strelka/rules - - user: 939 - - group: 939 - - clean: True -{% endif %} - {% else %} {{sls}}_state_not_allowed: diff --git a/salt/strelka/config.sls b/salt/strelka/config.sls index 1d0f75adf..929bef113 100644 --- a/salt/strelka/config.sls +++ b/salt/strelka/config.sls @@ -1,5 +1,5 @@ # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. @@ -21,6 +21,13 @@ strelkarulesdir: - group: 939 - makedirs: True +strelkareposdir: + file.directory: + - name: /opt/so/conf/strelka/repos + - user: 939 + - group: 939 + - makedirs: True + strelkadatadir: file.directory: - name: /nsm/strelka From 585147d1de66d700849a23057f7af85c97421433 Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Wed, 31 Jan 2024 10:39:47 -0700 Subject: [PATCH 03/17] Added so-detection mapping in elasticsearch --- salt/elasticsearch/defaults.yaml | 31 ++++- .../component/so/detection-mappings.json | 108 ++++++++++++++++++ .../component/so/detection-settings.json | 7 ++ 3 files changed, 145 insertions(+), 1 deletion(-) create mode 100644 salt/elasticsearch/templates/component/so/detection-mappings.json create mode 100644 salt/elasticsearch/templates/component/so/detection-settings.json diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index ce1bfb08d..480467129 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -117,6 +117,35 @@ elasticsearch: sort: field: '@timestamp' order: desc + so-detection: + index_sorting: false + index_template: + composed_of: + - detection-mappings + - detection-settings + index_patterns: + - so-detection* + priority: 500 + template: + mappings: + date_detection: false + dynamic_templates: + - strings_as_keyword: + mapping: + ignore_above: 1024 + type: keyword + match_mapping_type: string + settings: + index: + mapping: + total_fields: + limit: 1500 + number_of_replicas: 0 + number_of_shards: 1 + refresh_interval: 30s + sort: + field: '@timestamp' + order: desc so-common: close: 30 delete: 365 @@ -8909,7 +8938,7 @@ elasticsearch: actions: set_priority: priority: 50 - min_age: 30d + min_age: 30d so-logs-ti_otx_x_threat: index_sorting: false index_template: diff --git a/salt/elasticsearch/templates/component/so/detection-mappings.json b/salt/elasticsearch/templates/component/so/detection-mappings.json new file mode 100644 index 000000000..df53308f2 --- /dev/null +++ b/salt/elasticsearch/templates/component/so/detection-mappings.json @@ -0,0 +1,108 @@ +{ + "template": { + "mappings": { + "properties": { + "so_audit_doc_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "@timestamp": { + "type": "date" + }, + "so_kind": { + "ignore_above": 1024, + "type": "keyword" + }, + "so_operation": { + "ignore_above": 1024, + "type": "keyword" + }, + "so_detection": { + "properties": { + "publicId": { + "type": "text" + }, + "title": { + "type": "text" + }, + "severity": { + "ignore_above": 1024, + "type": "keyword" + }, + "author": { + "type": "text" + }, + "description": { + "type": "text" + }, + "content": { + "type": "text" + }, + "isEnabled": { + "type": "boolean" + }, + "isReporting": { + "type": "boolean" + }, + "isCommunity": { + "type": "boolean" + }, + "note": { + "type": "text" + }, + "engine": { + "ignore_above": 1024, + "type": "keyword" + }, + "overrides": { + "properties": { + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "isEnabled": { + "type": "boolean" + }, + "createdAt": { + "type": "date" + }, + "updatedAt": { + "type": "date" + }, + "regex": { + "type": "text" + }, + "value": { + "type": "text" + }, + "thresholdType": { + "ignore_above": 1024, + "type": "keyword" + }, + "track": { + "ignore_above": 1024, + "type": "keyword" + }, + "ip": { + "type": "text" + }, + "count": { + "type": "long" + }, + "seconds": { + "type": "long" + }, + "customFilter": { + "type": "text" + } + } + } + } + } + } + } + }, + "_meta": { + "ecs_version": "1.12.2" + } +} \ No newline at end of file diff --git a/salt/elasticsearch/templates/component/so/detection-settings.json b/salt/elasticsearch/templates/component/so/detection-settings.json new file mode 100644 index 000000000..7b0947a4c --- /dev/null +++ b/salt/elasticsearch/templates/component/so/detection-settings.json @@ -0,0 +1,7 @@ +{ + "template": {}, + "version": 1, + "_meta": { + "description": "default settings for common Security Onion Detections indices" + } +} \ No newline at end of file From 881d6b313e394a56fc7e64c3de3302a6a1719b3e Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 31 Jan 2024 17:04:11 -0500 Subject: [PATCH 04/17] Update VERSION - kilo --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 9cf89c6c7..7f2e97617 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4.50 +2.4.0-kilo From 49b5788ac14d4f79f6cac6b300e5f5cc6357172f Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Thu, 1 Feb 2024 07:21:49 -0500 Subject: [PATCH 05/17] add bindings --- salt/soc/enabled.sls | 1 + salt/strelka/backend/enabled.sls | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/soc/enabled.sls b/salt/soc/enabled.sls index 2661587f4..0eae0e80e 100644 --- a/salt/soc/enabled.sls +++ b/salt/soc/enabled.sls @@ -22,6 +22,7 @@ so-soc: - sobridge: - ipv4_address: {{ DOCKER.containers['so-soc'].ip }} - binds: + - /nsm/rules:/nsm/rules:rw #Need to tighten this up? - /nsm/soc/jobs:/opt/sensoroni/jobs:rw - /nsm/soc/uploads:/nsm/soc/uploads:rw - /opt/so/log/soc/:/opt/sensoroni/logs/:rw diff --git a/salt/strelka/backend/enabled.sls b/salt/strelka/backend/enabled.sls index fc56f4197..9ebb1a148 100644 --- a/salt/strelka/backend/enabled.sls +++ b/salt/strelka/backend/enabled.sls @@ -42,8 +42,8 @@ strelka_backend: {% endfor %} {% endif %} - restart_policy: on-failure - - watch: - - file: strelkarules + #- watch: + # - file: strelkarules delete_so-strelka-backend_so-status.disabled: file.uncomment: From fe196b56619ee240a81508a111e2e13427e3f382 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Thu, 1 Feb 2024 12:22:50 -0500 Subject: [PATCH 06/17] Add SOC Config for Detections --- salt/soc/defaults.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index c1b9470c8..fdbdfd6b2 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1002,6 +1002,11 @@ soc: jobDir: jobs kratos: hostUrl: + elastalertengine: + communityRulesImportFrequencySeconds: 180 + elastAlertRulesFolder: /opt/so/rules/elastalert + rulesFingerprintFile: /opt/so/conf/soc/sigma.fingerprint + sigmaRulePackages: all elastic: hostUrl: remoteHostUrls: [] @@ -1043,6 +1048,15 @@ soc: - rbac/custom_roles userFiles: - rbac/users_roles + strelkaengine: + compileYaraPythonScriptPath: /opt/so/conf/strelka/compile_yara.py + reposFolder: /nsm/rules/strelka/repos + rulesRepos: + - https://github.com/Security-Onion-Solutions/securityonion-yara + yaraRulesFolder: /opt/so/conf/strelka/rules + suricataengine: + communityRulesFile: /nsm/rules/suricata/emerging-all.rules + rulesFingerprintFile: /opt/so/conf/soc/emerging-all.fingerprint client: enableReverseLookup: false docsUrl: /docs/ From 8f81c9eb68b4c64dace40e065d91cf75b10019f0 Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Fri, 2 Feb 2024 11:49:58 -0700 Subject: [PATCH 07/17] Updating config for Detection(s) --- salt/soc/defaults.yaml | 52 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index fdbdfd6b2..7f6686431 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -64,7 +64,7 @@ soc: icon: fa-external-link-alt target: _blank links: - - 'https://{:sublime.url}/messages/{:sublime.message_group_id}' + - 'https://{:sublime.url}/messages/{:sublime.message_group_id}' eventFields: default: - soc_timestamp @@ -1756,3 +1756,53 @@ soc: - amber+strict - red customEnabled: false + detections: + viewEnabled: true + createLink: /detection/create + eventFetchLimit: 500 + eventItemsPerPage: 50 + groupFetchLimit: 50 + mostRecentlyUsedLimit: 5 + safeStringMaxLength: 100 + queryBaseFilter: '_index:"*:so-detection" AND so_kind:detection' + eventFields: + default: + - so_detection.title + - so_detection.isEnabled + - so_detection.engine + - "@timestamp" + queries: + - name: "All Detections" + query: "_id:*" + - name: "Local Rules" + query: "so_detection.isCommunity:false" + - name: "Enabled" + query: "so_detection.isEnabled:true" + - name: "Disabled" + query: "so_detection.isEnabled:false" + - name: "Suricata" + query: "so_detection.engine:suricata" + - name: "ElastAlert" + query: "so_detection.engine:elastalert" + - name: "Strelka" + query: "so_detection.engine:strelka" + detection: + presets: + severity: + customEnabled: false + labels: + - unknown + - informational + - low + - medium + - high + - critical + engine: + customEnabled: false + labels: + - suricata + - elastalert + - strelka + severityTranslations: + minor: low + major: high From 378c99ae8844d78bf6691aa2179cc898e2de17d4 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Fri, 2 Feb 2024 18:25:54 -0500 Subject: [PATCH 08/17] Fix bindings --- salt/soc/defaults.yaml | 8 ++++---- salt/soc/enabled.sls | 3 +++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 7f6686431..6811529bf 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1004,8 +1004,8 @@ soc: hostUrl: elastalertengine: communityRulesImportFrequencySeconds: 180 - elastAlertRulesFolder: /opt/so/rules/elastalert - rulesFingerprintFile: /opt/so/conf/soc/sigma.fingerprint + elastAlertRulesFolder: /opt/sensoroni/elastalert + rulesFingerprintFile: /opt/sensoroni/fingerprints/sigma.fingerprint sigmaRulePackages: all elastic: hostUrl: @@ -1053,10 +1053,10 @@ soc: reposFolder: /nsm/rules/strelka/repos rulesRepos: - https://github.com/Security-Onion-Solutions/securityonion-yara - yaraRulesFolder: /opt/so/conf/strelka/rules + yaraRulesFolder: /opt/sensoroni/yara suricataengine: communityRulesFile: /nsm/rules/suricata/emerging-all.rules - rulesFingerprintFile: /opt/so/conf/soc/emerging-all.fingerprint + rulesFingerprintFile: /opt/sensoroni/fingerprints/emerging-all.fingerprint client: enableReverseLookup: false docsUrl: /docs/ diff --git a/salt/soc/enabled.sls b/salt/soc/enabled.sls index 0eae0e80e..11f73e761 100644 --- a/salt/soc/enabled.sls +++ b/salt/soc/enabled.sls @@ -23,6 +23,9 @@ so-soc: - ipv4_address: {{ DOCKER.containers['so-soc'].ip }} - binds: - /nsm/rules:/nsm/rules:rw #Need to tighten this up? + - /opt/so/rules/yara:/opt/sensoroni/yara:rw + - /opt/so/rules/elastalert/rules:/opt/sensoroni/elastalert:rw + - /opt/so/conf/soc/fingerprints:/opt/sensoroni/fingerprints:rw - /nsm/soc/jobs:/opt/sensoroni/jobs:rw - /nsm/soc/uploads:/nsm/soc/uploads:rw - /opt/so/log/soc/:/opt/sensoroni/logs/:rw From b7b501d289a749c2d8fd0afe246cba9a2cec551b Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 7 Feb 2024 15:02:52 -0500 Subject: [PATCH 09/17] Add Sigma pipelines --- salt/soc/final_sigma_pipeline.yaml | 7 +++++++ salt/soc/so_sigma_pipeline.yaml | 18 ++++++++++++++++++ salt/soc/soc_soc.yaml | 13 +++++++++++++ 3 files changed, 38 insertions(+) create mode 100644 salt/soc/final_sigma_pipeline.yaml create mode 100644 salt/soc/so_sigma_pipeline.yaml diff --git a/salt/soc/final_sigma_pipeline.yaml b/salt/soc/final_sigma_pipeline.yaml new file mode 100644 index 000000000..656bfbb3e --- /dev/null +++ b/salt/soc/final_sigma_pipeline.yaml @@ -0,0 +1,7 @@ +name: Security Onion - Final Pipeline +priority: 95 +transformations: + - id: override_field_name_mapping + type: field_name_mapping + mapping: + FieldNameToOverride: NewFieldName diff --git a/salt/soc/so_sigma_pipeline.yaml b/salt/soc/so_sigma_pipeline.yaml new file mode 100644 index 000000000..a1c4d6d62 --- /dev/null +++ b/salt/soc/so_sigma_pipeline.yaml @@ -0,0 +1,18 @@ +name: Security Onion Baseline Pipeline +priority: 90 +transformations: + - id: baseline_field_name_mapping + type: field_name_mapping + mapping: + cs-method: http.method + c-uri: http.uri + c-useragent: http.useragent + cs-version: http.version + uid: user.uid + sid: rule.uuid + answer: answers + query: dns.query.name + src_ip: destination.ip.keyword + src_port: source.port + dst_ip: destination.ip.keyword + dst_port: destination.port \ No newline at end of file diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index 0dd39620b..fe672fe3e 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -32,6 +32,14 @@ soc: global: True advanced: True helpLink: soc-customization.html + final_sigma_pipeline__yaml: + title: Final Sigma Pipeline + description: Final Processing Pipeline for Sigma Rules + syntax: yaml + file: True + global: True + advanced: True + helpLink: soc-customization.html config: licenseKey: title: License Key @@ -62,6 +70,11 @@ soc: global: True advanced: True modules: + elastalertengine: + sigmaRulePackages: + description: 'One of the following: core | core+ | core++ | all' + global: True + advanced: False elastic: index: description: Comma-separated list of indices or index patterns (wildcard "*" supported) that SOC will search for records. From 7e3187c0b8f4bba0c992dbf7719a661bc9924735 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 7 Feb 2024 15:35:31 -0500 Subject: [PATCH 10/17] Fixup sigma pipelines --- salt/soc/config.sls | 16 ++++++++++++++++ salt/soc/enabled.sls | 2 ++ .../{ => files/soc}/final_sigma_pipeline.yaml | 0 salt/soc/{ => files/soc}/so_sigma_pipeline.yaml | 0 salt/soc/soc_soc.yaml | 2 +- 5 files changed, 19 insertions(+), 1 deletion(-) rename salt/soc/{ => files/soc}/final_sigma_pipeline.yaml (100%) rename salt/soc/{ => files/soc}/so_sigma_pipeline.yaml (100%) diff --git a/salt/soc/config.sls b/salt/soc/config.sls index 95135566b..549bf94cf 100644 --- a/salt/soc/config.sls +++ b/salt/soc/config.sls @@ -57,6 +57,22 @@ socmotd: - mode: 600 - template: jinja +socsigmafinalpipeline: + file.managed: + - name: /opt/so/conf/soc/sigma_final_pipeline.yaml + - source: salt://soc/files/soc/sigma_final_pipeline.yaml + - user: 939 + - group: 939 + - mode: 600 + +socsigmasopipeline: + file.managed: + - name: /opt/so/conf/soc/sigma_so_pipeline.yaml + - source: salt://soc/files/soc/sigma_so_pipeline.yaml + - user: 939 + - group: 939 + - mode: 600 + socbanner: file.managed: - name: /opt/so/conf/soc/banner.md diff --git a/salt/soc/enabled.sls b/salt/soc/enabled.sls index 11f73e761..535423179 100644 --- a/salt/soc/enabled.sls +++ b/salt/soc/enabled.sls @@ -32,6 +32,8 @@ so-soc: - /opt/so/conf/soc/soc.json:/opt/sensoroni/sensoroni.json:ro - /opt/so/conf/soc/motd.md:/opt/sensoroni/html/motd.md:ro - /opt/so/conf/soc/banner.md:/opt/sensoroni/html/login/banner.md:ro + - /opt/so/conf/soc/sigma_so_pipeline.yaml:/opt/sensoroni/sigma_so_pipeline.yaml:ro + - /opt/so/conf/soc/sigma_final_pipeline.yaml:/opt/sensoroni/sigma_final_pipeline.yaml:rw - /opt/so/conf/soc/custom.js:/opt/sensoroni/html/js/custom.js:ro - /opt/so/conf/soc/custom_roles:/opt/sensoroni/rbac/custom_roles:ro - /opt/so/conf/soc/soc_users_roles:/opt/sensoroni/rbac/users_roles:rw diff --git a/salt/soc/final_sigma_pipeline.yaml b/salt/soc/files/soc/final_sigma_pipeline.yaml similarity index 100% rename from salt/soc/final_sigma_pipeline.yaml rename to salt/soc/files/soc/final_sigma_pipeline.yaml diff --git a/salt/soc/so_sigma_pipeline.yaml b/salt/soc/files/soc/so_sigma_pipeline.yaml similarity index 100% rename from salt/soc/so_sigma_pipeline.yaml rename to salt/soc/files/soc/so_sigma_pipeline.yaml diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index fe672fe3e..f413b5c73 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -32,7 +32,7 @@ soc: global: True advanced: True helpLink: soc-customization.html - final_sigma_pipeline__yaml: + sigma_final_pipeline__yaml: title: Final Sigma Pipeline description: Final Processing Pipeline for Sigma Rules syntax: yaml From 81a3e95914d602386a11e2f5289a98fe59f97ece Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 7 Feb 2024 16:42:16 -0500 Subject: [PATCH 11/17] Fixup sigma pipelines --- .../soc/{final_sigma_pipeline.yaml => sigma_final_pipeline.yaml} | 0 .../files/soc/{so_sigma_pipeline.yaml => sigma_so_pipeline.yaml} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename salt/soc/files/soc/{final_sigma_pipeline.yaml => sigma_final_pipeline.yaml} (100%) rename salt/soc/files/soc/{so_sigma_pipeline.yaml => sigma_so_pipeline.yaml} (100%) diff --git a/salt/soc/files/soc/final_sigma_pipeline.yaml b/salt/soc/files/soc/sigma_final_pipeline.yaml similarity index 100% rename from salt/soc/files/soc/final_sigma_pipeline.yaml rename to salt/soc/files/soc/sigma_final_pipeline.yaml diff --git a/salt/soc/files/soc/so_sigma_pipeline.yaml b/salt/soc/files/soc/sigma_so_pipeline.yaml similarity index 100% rename from salt/soc/files/soc/so_sigma_pipeline.yaml rename to salt/soc/files/soc/sigma_so_pipeline.yaml From 29174566f3f7a8307a92eaac2073f108eddedc57 Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Thu, 8 Feb 2024 09:44:56 -0700 Subject: [PATCH 12/17] WIP: Updated Detection Mappings, Changed Engine to Language Detection mappings updated to include the removal of Note and the addition of Tags, Ruleset, and Language. SOC defaults updated to use language based queries rather than engine and show the language column instead of the engine column in results. --- .../component/so/detection-mappings.json | 10 +++++++++- salt/soc/defaults.yaml | 18 +++++++++--------- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/salt/elasticsearch/templates/component/so/detection-mappings.json b/salt/elasticsearch/templates/component/so/detection-mappings.json index df53308f2..596236703 100644 --- a/salt/elasticsearch/templates/component/so/detection-mappings.json +++ b/salt/elasticsearch/templates/component/so/detection-mappings.json @@ -47,13 +47,21 @@ "isCommunity": { "type": "boolean" }, - "note": { + "tags": { "type": "text" }, + "ruleset": { + "ignore_above": 1024, + "type": "keyword" + }, "engine": { "ignore_above": 1024, "type": "keyword" }, + "language": { + "ignore_above": 1024, + "type": "keyword" + }, "overrides": { "properties": { "type": { diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 6811529bf..cc6b417e6 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1769,7 +1769,7 @@ soc: default: - so_detection.title - so_detection.isEnabled - - so_detection.engine + - so_detection.language - "@timestamp" queries: - name: "All Detections" @@ -1781,11 +1781,11 @@ soc: - name: "Disabled" query: "so_detection.isEnabled:false" - name: "Suricata" - query: "so_detection.engine:suricata" - - name: "ElastAlert" - query: "so_detection.engine:elastalert" - - name: "Strelka" - query: "so_detection.engine:strelka" + query: "so_detection.language:suricata" + - name: "Sigma" + query: "so_detection.language:sigma" + - name: "Yara" + query: "so_detection.language:yara" detection: presets: severity: @@ -1797,12 +1797,12 @@ soc: - medium - high - critical - engine: + language: customEnabled: false labels: - suricata - - elastalert - - strelka + - sigma + - yara severityTranslations: minor: low major: high From 64f6d0fba93180f0627b56b5adeb822afa8c5193 Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Fri, 9 Feb 2024 14:20:07 -0700 Subject: [PATCH 13/17] Updated Detection's ES Mappings Detection's now have a License field and the Comment model is defined now. --- .../component/so/detection-mappings.json | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/salt/elasticsearch/templates/component/so/detection-mappings.json b/salt/elasticsearch/templates/component/so/detection-mappings.json index 596236703..4efd2f73f 100644 --- a/salt/elasticsearch/templates/component/so/detection-mappings.json +++ b/salt/elasticsearch/templates/component/so/detection-mappings.json @@ -62,6 +62,10 @@ "ignore_above": 1024, "type": "keyword" }, + "license": { + "ignore_above": 1024, + "type": "keyword" + }, "overrides": { "properties": { "type": { @@ -106,6 +110,24 @@ } } } + }, + "so_comment": { + "properties": { + "createTime": { + "type": "date" + }, + "detectionId": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "type": "text" + }, + "userId": { + "ignore_above": 1024, + "type": "keyword" + } + } } } } From 510226944077b5ac460869d2eced99c0438ba169 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Mon, 12 Feb 2024 16:44:54 -0500 Subject: [PATCH 14/17] Update defaults --- salt/idstools/soc_idstools.yaml | 2 +- salt/soc/defaults.yaml | 2 +- salt/soc/soc_soc.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/idstools/soc_idstools.yaml b/salt/idstools/soc_idstools.yaml index 634f68803..7cf11dba4 100644 --- a/salt/idstools/soc_idstools.yaml +++ b/salt/idstools/soc_idstools.yaml @@ -8,7 +8,7 @@ idstools: global: True helpLink: rules.html ruleset: - description: Defines the ruleset you want to run. Options are ETOPEN or ETPRO. + description: Defines the ruleset you want to run. Options are ETOPEN or ETPRO. -- WARNING -- Changing the ruleset will remove all existing Suricata rules of the previous ruleset and their associated overrides.' global: True regex: ETPRO\b|ETOPEN\b helpLink: rules.html diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index cc6b417e6..fd2eaf8c0 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1050,7 +1050,7 @@ soc: - rbac/users_roles strelkaengine: compileYaraPythonScriptPath: /opt/so/conf/strelka/compile_yara.py - reposFolder: /nsm/rules/strelka/repos + reposFolder: /nsm/rules/yara/repos rulesRepos: - https://github.com/Security-Onion-Solutions/securityonion-yara yaraRulesFolder: /opt/sensoroni/yara diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index f413b5c73..fe0458820 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -72,7 +72,7 @@ soc: modules: elastalertengine: sigmaRulePackages: - description: 'One of the following: core | core+ | core++ | all' + description: 'Defines the Sigma Community Ruleset you want to run: core | core+ | core++ | all. -- WARNING -- Changing the ruleset will remove all existing Sigma rules of the previous ruleset and their associated overrides.' global: True advanced: False elastic: From ea80469c2db1bc690e26e4a7e5cf5c1afd44bc3d Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Mon, 12 Feb 2024 19:39:55 -0500 Subject: [PATCH 15/17] Detection Default queries --- salt/soc/defaults.yaml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index fd2eaf8c0..29cd7e1ac 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1770,21 +1770,23 @@ soc: - so_detection.title - so_detection.isEnabled - so_detection.language - - "@timestamp" + - so_detection.severity queries: - name: "All Detections" query: "_id:*" - - name: "Local Rules" + - name: "Custom Detections" query: "so_detection.isCommunity:false" - - name: "Enabled" + - name: "All Detections - Enabled" query: "so_detection.isEnabled:true" - - name: "Disabled" + - name: "All Detections - Disabled" query: "so_detection.isEnabled:false" - - name: "Suricata" + - name: "Detection Type - Suricata (NIDS)" query: "so_detection.language:suricata" - - name: "Sigma" + - name: "Detection Type - Sigma - All" query: "so_detection.language:sigma" - - name: "Yara" + - name: "Detection Type - Sigma - Windows" + query: 'so_detection.language:sigma AND so_detection.content: "*product: windows*"' + - name: "Detection Type - Yara (Strelka)" query: "so_detection.language:yara" detection: presets: From 0c6c6ba2d5d2e529ab6239f53ea6bd38d8a13446 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Tue, 13 Feb 2024 13:38:43 -0500 Subject: [PATCH 16/17] Various UI tweaks --- salt/soc/config.sls | 9 ++++++++- salt/soc/defaults.yaml | 6 +++--- salt/soc/enabled.sls | 2 +- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/salt/soc/config.sls b/salt/soc/config.sls index 549bf94cf..e4dad8df2 100644 --- a/salt/soc/config.sls +++ b/salt/soc/config.sls @@ -9,9 +9,16 @@ include: - manager.sync_es_users +socdirtest: + file.directory: + - name: /opt/so/rules/elastalert/rules + - user: 939 + - group: 939 + - makedirs: True + socdir: file.directory: - - name: /opt/so/conf/soc + - name: /opt/so/conf/soc/fingerprints - user: 939 - group: 939 - makedirs: True diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 29cd7e1ac..c060698b4 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1006,7 +1006,7 @@ soc: communityRulesImportFrequencySeconds: 180 elastAlertRulesFolder: /opt/sensoroni/elastalert rulesFingerprintFile: /opt/sensoroni/fingerprints/sigma.fingerprint - sigmaRulePackages: all + sigmaRulePackages: core elastic: hostUrl: remoteHostUrls: [] @@ -1050,10 +1050,10 @@ soc: - rbac/users_roles strelkaengine: compileYaraPythonScriptPath: /opt/so/conf/strelka/compile_yara.py - reposFolder: /nsm/rules/yara/repos + reposFolder: /opt/sensoroni/yara/repos rulesRepos: - https://github.com/Security-Onion-Solutions/securityonion-yara - yaraRulesFolder: /opt/sensoroni/yara + yaraRulesFolder: /opt/sensoroni/yara/rules suricataengine: communityRulesFile: /nsm/rules/suricata/emerging-all.rules rulesFingerprintFile: /opt/sensoroni/fingerprints/emerging-all.fingerprint diff --git a/salt/soc/enabled.sls b/salt/soc/enabled.sls index 535423179..7c04da825 100644 --- a/salt/soc/enabled.sls +++ b/salt/soc/enabled.sls @@ -23,7 +23,7 @@ so-soc: - ipv4_address: {{ DOCKER.containers['so-soc'].ip }} - binds: - /nsm/rules:/nsm/rules:rw #Need to tighten this up? - - /opt/so/rules/yara:/opt/sensoroni/yara:rw + - /opt/so/conf/strelka:/opt/sensoroni/yara:rw - /opt/so/rules/elastalert/rules:/opt/sensoroni/elastalert:rw - /opt/so/conf/soc/fingerprints:/opt/sensoroni/fingerprints:rw - /nsm/soc/jobs:/opt/sensoroni/jobs:rw From 0d297274c8fc4f278e7b9e8530bdd39856425fb9 Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Tue, 13 Feb 2024 12:53:18 -0700 Subject: [PATCH 17/17] DetectionComment Mapping Defined --- .../templates/component/so/detection-mappings.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/elasticsearch/templates/component/so/detection-mappings.json b/salt/elasticsearch/templates/component/so/detection-mappings.json index 4efd2f73f..9b68421e7 100644 --- a/salt/elasticsearch/templates/component/so/detection-mappings.json +++ b/salt/elasticsearch/templates/component/so/detection-mappings.json @@ -111,7 +111,7 @@ } } }, - "so_comment": { + "so_detectioncomment": { "properties": { "createTime": { "type": "date" @@ -120,7 +120,7 @@ "ignore_above": 1024, "type": "keyword" }, - "description": { + "value": { "type": "text" }, "userId": {