diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 03cd6d519..f4d8c8a95 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -198,6 +198,35 @@ elasticsearch: sort: field: '@timestamp' order: desc + so-detection: + index_sorting: false + index_template: + composed_of: + - detection-mappings + - detection-settings + index_patterns: + - so-detection* + priority: 500 + template: + mappings: + date_detection: false + dynamic_templates: + - strings_as_keyword: + mapping: + ignore_above: 1024 + type: keyword + match_mapping_type: string + settings: + index: + mapping: + total_fields: + limit: 1500 + number_of_replicas: 0 + number_of_shards: 1 + refresh_interval: 30s + sort: + field: '@timestamp' + order: desc so-common: close: 30 delete: 365 @@ -8990,7 +9019,7 @@ elasticsearch: actions: set_priority: priority: 50 - min_age: 30d + min_age: 30d so-logs-ti_otx_x_threat: index_sorting: false index_template: diff --git a/salt/elasticsearch/templates/component/so/detection-mappings.json b/salt/elasticsearch/templates/component/so/detection-mappings.json new file mode 100644 index 000000000..9b68421e7 --- /dev/null +++ b/salt/elasticsearch/templates/component/so/detection-mappings.json @@ -0,0 +1,138 @@ +{ + "template": { + "mappings": { + "properties": { + "so_audit_doc_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "@timestamp": { + "type": "date" + }, + "so_kind": { + "ignore_above": 1024, + "type": "keyword" + }, + "so_operation": { + "ignore_above": 1024, + "type": "keyword" + }, + "so_detection": { + "properties": { + "publicId": { + "type": "text" + }, + "title": { + "type": "text" + }, + "severity": { + "ignore_above": 1024, + "type": "keyword" + }, + "author": { + "type": "text" + }, + "description": { + "type": "text" + }, + "content": { + "type": "text" + }, + "isEnabled": { + "type": "boolean" + }, + "isReporting": { + "type": "boolean" + }, + "isCommunity": { + "type": "boolean" + }, + "tags": { + "type": "text" + }, + "ruleset": { + "ignore_above": 1024, + "type": "keyword" + }, + "engine": { + "ignore_above": 1024, + "type": "keyword" + }, + "language": { + "ignore_above": 1024, + "type": "keyword" + }, + "license": { + "ignore_above": 1024, + "type": "keyword" + }, + "overrides": { + "properties": { + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "isEnabled": { + "type": "boolean" + }, + "createdAt": { + "type": "date" + }, + "updatedAt": { + "type": "date" + }, + "regex": { + "type": "text" + }, + "value": { + "type": "text" + }, + "thresholdType": { + "ignore_above": 1024, + "type": "keyword" + }, + "track": { + "ignore_above": 1024, + "type": "keyword" + }, + "ip": { + "type": "text" + }, + "count": { + "type": "long" + }, + "seconds": { + "type": "long" + }, + "customFilter": { + "type": "text" + } + } + } + } + }, + "so_detectioncomment": { + "properties": { + "createTime": { + "type": "date" + }, + "detectionId": { + "ignore_above": 1024, + "type": "keyword" + }, + "value": { + "type": "text" + }, + "userId": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + } + }, + "_meta": { + "ecs_version": "1.12.2" + } +} \ No newline at end of file diff --git a/salt/elasticsearch/templates/component/so/detection-settings.json b/salt/elasticsearch/templates/component/so/detection-settings.json new file mode 100644 index 000000000..7b0947a4c --- /dev/null +++ b/salt/elasticsearch/templates/component/so/detection-settings.json @@ -0,0 +1,7 @@ +{ + "template": {}, + "version": 1, + "_meta": { + "description": "default settings for common Security Onion Detections indices" + } +} \ No newline at end of file diff --git a/salt/idstools/soc_idstools.yaml b/salt/idstools/soc_idstools.yaml index 634f68803..f8ec3b8b6 100644 --- a/salt/idstools/soc_idstools.yaml +++ b/salt/idstools/soc_idstools.yaml @@ -8,7 +8,7 @@ idstools: global: True helpLink: rules.html ruleset: - description: Defines the ruleset you want to run. Options are ETOPEN or ETPRO. + description: 'Defines the ruleset you want to run. Options are ETOPEN or ETPRO. WARNING! Changing the ruleset will remove all existing Suricata rules of the previous ruleset and their associated overrides. This removal cannot be undone.' global: True regex: ETPRO\b|ETOPEN\b helpLink: rules.html diff --git a/salt/manager/init.sls b/salt/manager/init.sls index 23ef189b5..51590a6ec 100644 --- a/salt/manager/init.sls +++ b/salt/manager/init.sls @@ -1,5 +1,5 @@ # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. @@ -61,7 +61,7 @@ manager_sbin: - user: 939 - group: 939 - file_mode: 755 - - exclude_pat: + - exclude_pat: - "*_test.py" yara_update_scripts: @@ -103,55 +103,6 @@ rules_dir: - group: socore - makedirs: True -{% if STRELKAMERGED.rules.enabled %} - -strelkarepos: - file.managed: - - name: /opt/so/conf/strelka/repos.txt - - source: salt://strelka/rules/repos.txt.jinja - - template: jinja - - defaults: - STRELKAREPOS: {{ STRELKAMERGED.rules.repos }} - - makedirs: True - -strelka-yara-update: - {% if MANAGERMERGED.reposync.enabled and not GLOBALS.airgap %} - cron.present: - {% else %} - cron.absent: - {% endif %} - - user: socore - - name: '/usr/sbin/so-yara-update >> /opt/so/log/yarasync/yara-update.log 2>&1' - - identifier: strelka-yara-update - - hour: '7' - - minute: '1' - -strelka-yara-download: - {% if MANAGERMERGED.reposync.enabled and not GLOBALS.airgap %} - cron.present: - {% else %} - cron.absent: - {% endif %} - - user: socore - - name: '/usr/sbin/so-yara-download >> /opt/so/log/yarasync/yara-download.log 2>&1' - - identifier: strelka-yara-download - - hour: '7' - - minute: '1' - -{% if not GLOBALS.airgap %} -update_yara_rules: - cmd.run: - - name: /usr/sbin/so-yara-update - - onchanges: - - file: yara_update_scripts - -download_yara_rules: - cmd.run: - - name: /usr/sbin/so-yara-download - - onchanges: - - file: yara_update_scripts -{% endif %} -{% endif %} {% else %} {{sls}}_state_not_allowed: diff --git a/salt/manager/tools/sbin_jinja/so-yara-download b/salt/manager/tools/sbin_jinja/so-yara-download deleted file mode 100644 index aa9576253..000000000 --- a/salt/manager/tools/sbin_jinja/so-yara-download +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/bash -NOROOT=1 -. /usr/sbin/so-common - -{%- set proxy = salt['pillar.get']('manager:proxy') %} -{%- set noproxy = salt['pillar.get']('manager:no_proxy', '') %} - -# Download the rules from the internet -{%- if proxy %} -export http_proxy={{ proxy }} -export https_proxy={{ proxy }} -export no_proxy="{{ noproxy }}" -{%- endif %} - -repos="/opt/so/conf/strelka/repos.txt" -output_dir=/nsm/rules/yara -gh_status=$(curl -s -o /dev/null -w "%{http_code}" https://github.com) -clone_dir="/tmp" -if [ "$gh_status" == "200" ] || [ "$gh_status" == "301" ]; then - - while IFS= read -r repo; do - if ! $(echo "$repo" | grep -qE '^#'); then - # Remove old repo if existing bc of previous error condition or unexpected disruption - repo_name=`echo $repo | awk -F '/' '{print $NF}'` - [ -d $output_dir/$repo_name ] && rm -rf $output_dir/$repo_name - - # Clone repo and make appropriate directories for rules - git clone $repo $clone_dir/$repo_name - echo "Analyzing rules from $clone_dir/$repo_name..." - mkdir -p $output_dir/$repo_name - # Ensure a copy of the license is available for the rules - [ -f $clone_dir/$repo_name/LICENSE ] && cp $clone_dir/$repo_name/LICENSE $output_dir/$repo_name - - # Copy over rules - for i in $(find $clone_dir/$repo_name -name "*.yar*"); do - rule_name=$(echo $i | awk -F '/' '{print $NF}') - cp $i $output_dir/$repo_name - done - rm -rf $clone_dir/$repo_name - fi - done < $repos - - echo "Done!" - -/usr/sbin/so-yara-update - -else - echo "Server returned $gh_status status code." - echo "No connectivity to Github...exiting..." - exit 1 -fi diff --git a/salt/manager/tools/sbin_jinja/so-yara-update b/salt/manager/tools/sbin_jinja/so-yara-update deleted file mode 100755 index 07c940f47..000000000 --- a/salt/manager/tools/sbin_jinja/so-yara-update +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -NOROOT=1 -. /usr/sbin/so-common - -echo "Starting to check for yara rule updates at $(date)..." - -newcounter=0 -excludedcounter=0 -excluded_rules=({{ EXCLUDEDRULES | join(' ') }}) - -# Pull down the SO Rules -SORULEDIR=/nsm/rules/yara -OUTPUTDIR=/opt/so/saltstack/local/salt/strelka/rules - -mkdir -p $OUTPUTDIR -# remove all rules prior to copy so we can clear out old rules -rm -f $OUTPUTDIR/* - -for i in $(find $SORULEDIR -name "*.yar" -o -name "*.yara"); do - rule_name=$(echo $i | awk -F '/' '{print $NF}') - if [[ ! "${excluded_rules[*]}" =~ ${rule_name} ]]; then - echo "Adding rule: $rule_name..." - cp $i $OUTPUTDIR/$rule_name - ((newcounter++)) - else - echo "Excluding rule: $rule_name..." - ((excludedcounter++)) - fi -done - -if [ "$newcounter" -gt 0 ] || [ "$excludedcounter" -gt 0 ];then - echo "$newcounter rules added." - echo "$excludedcounter rule(s) excluded." -fi - -echo "Finished rule updates at $(date)..." diff --git a/salt/soc/config.sls b/salt/soc/config.sls index 902d82ec7..e4dad8df2 100644 --- a/salt/soc/config.sls +++ b/salt/soc/config.sls @@ -9,9 +9,16 @@ include: - manager.sync_es_users +socdirtest: + file.directory: + - name: /opt/so/rules/elastalert/rules + - user: 939 + - group: 939 + - makedirs: True + socdir: file.directory: - - name: /opt/so/conf/soc + - name: /opt/so/conf/soc/fingerprints - user: 939 - group: 939 - makedirs: True @@ -57,6 +64,22 @@ socmotd: - mode: 600 - template: jinja +socsigmafinalpipeline: + file.managed: + - name: /opt/so/conf/soc/sigma_final_pipeline.yaml + - source: salt://soc/files/soc/sigma_final_pipeline.yaml + - user: 939 + - group: 939 + - mode: 600 + +socsigmasopipeline: + file.managed: + - name: /opt/so/conf/soc/sigma_so_pipeline.yaml + - source: salt://soc/files/soc/sigma_so_pipeline.yaml + - user: 939 + - group: 939 + - mode: 600 + socbanner: file.managed: - name: /opt/so/conf/soc/banner.md @@ -114,6 +137,13 @@ socuploaddir: - group: 939 - makedirs: True +socsigmarepo: + file.directory: + - name: /opt/so/rules + - user: 939 + - group: 939 + - mode: 775 + {% else %} {{sls}}_state_not_allowed: diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index dc836a9d7..2c15fe996 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -64,7 +64,7 @@ soc: icon: fa-external-link-alt target: _blank links: - - 'https://{:sublime.url}/messages/{:sublime.message_group_id}' + - 'https://{:sublime.url}/messages/{:sublime.message_group_id}' - name: actionProcessAncestors description: actionProcessAncestorsHelp icon: fa-people-roof @@ -1008,6 +1008,13 @@ soc: jobDir: jobs kratos: hostUrl: + elastalertengine: + communityRulesImportFrequencySeconds: 180 + elastAlertRulesFolder: /opt/sensoroni/elastalert + rulesFingerprintFile: /opt/sensoroni/fingerprints/sigma.fingerprint + sigmaRulePackages: + - core + - emerging_threats_addon elastic: hostUrl: remoteHostUrls: [] @@ -1049,6 +1056,15 @@ soc: - rbac/custom_roles userFiles: - rbac/users_roles + strelkaengine: + compileYaraPythonScriptPath: /opt/so/conf/strelka/compile_yara.py + reposFolder: /opt/sensoroni/yara/repos + rulesRepos: + - https://github.com/Security-Onion-Solutions/securityonion-yara + yaraRulesFolder: /opt/sensoroni/yara/rules + suricataengine: + communityRulesFile: /nsm/rules/suricata/emerging-all.rules + rulesFingerprintFile: /opt/sensoroni/fingerprints/emerging-all.fingerprint client: enableReverseLookup: false docsUrl: /docs/ @@ -1754,3 +1770,55 @@ soc: - amber+strict - red customEnabled: false + detections: + viewEnabled: true + createLink: /detection/create + eventFetchLimit: 500 + eventItemsPerPage: 50 + groupFetchLimit: 50 + mostRecentlyUsedLimit: 5 + safeStringMaxLength: 100 + queryBaseFilter: '_index:"*:so-detection" AND so_kind:detection' + eventFields: + default: + - so_detection.title + - so_detection.isEnabled + - so_detection.language + - so_detection.severity + queries: + - name: "All Detections" + query: "_id:*" + - name: "Custom Detections" + query: "so_detection.isCommunity:false" + - name: "All Detections - Enabled" + query: "so_detection.isEnabled:true" + - name: "All Detections - Disabled" + query: "so_detection.isEnabled:false" + - name: "Detection Type - Suricata (NIDS)" + query: "so_detection.language:suricata" + - name: "Detection Type - Sigma - All" + query: "so_detection.language:sigma" + - name: "Detection Type - Sigma - Windows" + query: 'so_detection.language:sigma AND so_detection.content: "*product: windows*"' + - name: "Detection Type - Yara (Strelka)" + query: "so_detection.language:yara" + detection: + presets: + severity: + customEnabled: false + labels: + - unknown + - informational + - low + - medium + - high + - critical + language: + customEnabled: false + labels: + - suricata + - sigma + - yara + severityTranslations: + minor: low + major: high diff --git a/salt/soc/enabled.sls b/salt/soc/enabled.sls index 2661587f4..7c04da825 100644 --- a/salt/soc/enabled.sls +++ b/salt/soc/enabled.sls @@ -22,12 +22,18 @@ so-soc: - sobridge: - ipv4_address: {{ DOCKER.containers['so-soc'].ip }} - binds: + - /nsm/rules:/nsm/rules:rw #Need to tighten this up? + - /opt/so/conf/strelka:/opt/sensoroni/yara:rw + - /opt/so/rules/elastalert/rules:/opt/sensoroni/elastalert:rw + - /opt/so/conf/soc/fingerprints:/opt/sensoroni/fingerprints:rw - /nsm/soc/jobs:/opt/sensoroni/jobs:rw - /nsm/soc/uploads:/nsm/soc/uploads:rw - /opt/so/log/soc/:/opt/sensoroni/logs/:rw - /opt/so/conf/soc/soc.json:/opt/sensoroni/sensoroni.json:ro - /opt/so/conf/soc/motd.md:/opt/sensoroni/html/motd.md:ro - /opt/so/conf/soc/banner.md:/opt/sensoroni/html/login/banner.md:ro + - /opt/so/conf/soc/sigma_so_pipeline.yaml:/opt/sensoroni/sigma_so_pipeline.yaml:ro + - /opt/so/conf/soc/sigma_final_pipeline.yaml:/opt/sensoroni/sigma_final_pipeline.yaml:rw - /opt/so/conf/soc/custom.js:/opt/sensoroni/html/js/custom.js:ro - /opt/so/conf/soc/custom_roles:/opt/sensoroni/rbac/custom_roles:ro - /opt/so/conf/soc/soc_users_roles:/opt/sensoroni/rbac/users_roles:rw diff --git a/salt/soc/files/bin/compile_yara.py b/salt/soc/files/bin/compile_yara.py new file mode 100644 index 000000000..43c8b1a09 --- /dev/null +++ b/salt/soc/files/bin/compile_yara.py @@ -0,0 +1,14 @@ +import os +import yara +import glob +import sys + +def compile_yara_rules(rules_dir: str) -> None: + compiled_rules_path: str = os.path.join(rules_dir, "rules.yar.compiled") + rule_files: list[str] = glob.glob(os.path.join(rules_dir, '**/*.yar'), recursive=True) + + if rule_files: + rules: yara.Rules = yara.compile(filepaths={os.path.basename(f): f for f in rule_files}) + rules.save(compiled_rules_path) + +compile_yara_rules(sys.argv[1]) diff --git a/salt/soc/files/soc/sigma_final_pipeline.yaml b/salt/soc/files/soc/sigma_final_pipeline.yaml new file mode 100644 index 000000000..656bfbb3e --- /dev/null +++ b/salt/soc/files/soc/sigma_final_pipeline.yaml @@ -0,0 +1,7 @@ +name: Security Onion - Final Pipeline +priority: 95 +transformations: + - id: override_field_name_mapping + type: field_name_mapping + mapping: + FieldNameToOverride: NewFieldName diff --git a/salt/soc/files/soc/sigma_so_pipeline.yaml b/salt/soc/files/soc/sigma_so_pipeline.yaml new file mode 100644 index 000000000..a1c4d6d62 --- /dev/null +++ b/salt/soc/files/soc/sigma_so_pipeline.yaml @@ -0,0 +1,18 @@ +name: Security Onion Baseline Pipeline +priority: 90 +transformations: + - id: baseline_field_name_mapping + type: field_name_mapping + mapping: + cs-method: http.method + c-uri: http.uri + c-useragent: http.useragent + cs-version: http.version + uid: user.uid + sid: rule.uuid + answer: answers + query: dns.query.name + src_ip: destination.ip.keyword + src_port: source.port + dst_ip: destination.ip.keyword + dst_port: destination.port \ No newline at end of file diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index 0dd39620b..fdfb09733 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -32,6 +32,14 @@ soc: global: True advanced: True helpLink: soc-customization.html + sigma_final_pipeline__yaml: + title: Final Sigma Pipeline + description: Final Processing Pipeline for Sigma Rules + syntax: yaml + file: True + global: True + advanced: True + helpLink: soc-customization.html config: licenseKey: title: License Key @@ -62,6 +70,11 @@ soc: global: True advanced: True modules: + elastalertengine: + sigmaRulePackages: + description: 'Defines the Sigma Community Ruleset you want to run. One of these (core | core+ | core++ | all ) as well as an optional Add-on (emerging_threats_addon). WARNING! Changing the ruleset will remove all existing Sigma rules of the previous ruleset and their associated overrides. This removal cannot be undone.' + global: True + advanced: False elastic: index: description: Comma-separated list of indices or index patterns (wildcard "*" supported) that SOC will search for records. diff --git a/salt/strelka/backend/config.sls b/salt/strelka/backend/config.sls index d51debb1b..b39e06ac8 100644 --- a/salt/strelka/backend/config.sls +++ b/salt/strelka/backend/config.sls @@ -50,16 +50,6 @@ backend_taste: - user: 939 - group: 939 -{% if STRELKAMERGED.rules.enabled %} -strelkarules: - file.recurse: - - name: /opt/so/conf/strelka/rules - - source: salt://strelka/rules - - user: 939 - - group: 939 - - clean: True -{% endif %} - {% else %} {{sls}}_state_not_allowed: diff --git a/salt/strelka/backend/enabled.sls b/salt/strelka/backend/enabled.sls index fc56f4197..9ebb1a148 100644 --- a/salt/strelka/backend/enabled.sls +++ b/salt/strelka/backend/enabled.sls @@ -42,8 +42,8 @@ strelka_backend: {% endfor %} {% endif %} - restart_policy: on-failure - - watch: - - file: strelkarules + #- watch: + # - file: strelkarules delete_so-strelka-backend_so-status.disabled: file.uncomment: diff --git a/salt/strelka/config.sls b/salt/strelka/config.sls index 1d0f75adf..929bef113 100644 --- a/salt/strelka/config.sls +++ b/salt/strelka/config.sls @@ -1,5 +1,5 @@ # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. @@ -21,6 +21,13 @@ strelkarulesdir: - group: 939 - makedirs: True +strelkareposdir: + file.directory: + - name: /opt/so/conf/strelka/repos + - user: 939 + - group: 939 + - makedirs: True + strelkadatadir: file.directory: - name: /nsm/strelka