Merge pull request #12412 from Security-Onion-Solutions/kilo

Initial Support for Detections Module

salt/elasticsearch/defaults.yaml

@@ -198,6 +198,35 @@ elasticsearch:
               sort:
                 field: '@timestamp'
                 order: desc
+    so-detection:
+      index_sorting: false
+      index_template:
+        composed_of:
+        - detection-mappings
+        - detection-settings
+        index_patterns:
+        - so-detection*
+        priority: 500
+        template:
+          mappings:
+            date_detection: false
+            dynamic_templates:
+            - strings_as_keyword:
+                mapping:
+                  ignore_above: 1024
+                  type: keyword
+                match_mapping_type: string
+          settings:
+            index:
+              mapping:
+                total_fields:
+                  limit: 1500
+              number_of_replicas: 0
+              number_of_shards: 1
+              refresh_interval: 30s
+              sort:
+                field: '@timestamp'
+                order: desc
     so-common:
       close: 30
       delete: 365
@@ -8990,7 +9019,7 @@ elasticsearch:
             actions:
               set_priority:
                 priority: 50
-            min_age: 30d    
+            min_age: 30d
     so-logs-ti_otx_x_threat:
       index_sorting: false
       index_template:

salt/elasticsearch/templates/component/so/detection-mappings.json

@@ -0,0 +1,138 @@
+{
+  "template": {
+    "mappings": {
+      "properties": {
+        "so_audit_doc_id": {
+          "ignore_above": 1024,
+          "type": "keyword"
+        },
+        "@timestamp": {
+          "type": "date"
+        },
+        "so_kind": {
+          "ignore_above": 1024,
+          "type": "keyword"
+        },
+        "so_operation": {
+          "ignore_above": 1024,
+          "type": "keyword"
+        },
+        "so_detection": {
+          "properties": {
+            "publicId": {
+              "type": "text"
+            },
+            "title": {
+              "type": "text"
+            },
+            "severity": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "author": {
+              "type": "text"
+            },
+            "description": {
+              "type": "text"
+            },
+            "content": {
+              "type": "text"
+            },
+            "isEnabled": {
+              "type": "boolean"
+            },
+            "isReporting": {
+              "type": "boolean"
+            },
+            "isCommunity": {
+              "type": "boolean"
+            },
+            "tags": {
+              "type": "text"
+            },
+            "ruleset": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "engine": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "language": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "license": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "overrides": {
+              "properties": {
+                "type": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "isEnabled": {
+                  "type": "boolean"
+                },
+                "createdAt": {
+                  "type": "date"
+                },
+                "updatedAt": {
+                  "type": "date"
+                },
+                "regex": {
+                  "type": "text"
+                },
+                "value": {
+                  "type": "text"
+                },
+                "thresholdType": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "track": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "ip": {
+                  "type": "text"
+                },
+                "count": {
+                  "type": "long"
+                },
+                "seconds": {
+                  "type": "long"
+                },
+                "customFilter": {
+                  "type": "text"
+                }
+              }
+            }
+          }
+        },
+        "so_detectioncomment": {
+          "properties": {
+            "createTime": {
+              "type": "date"
+            },
+            "detectionId": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "value": {
+              "type": "text"
+            },
+            "userId": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            }
+          }
+        }
+      }
+    }
+  },
+  "_meta": {
+    "ecs_version": "1.12.2"
+  }
+}

salt/elasticsearch/templates/component/so/detection-settings.json

@@ -0,0 +1,7 @@
+{
+  "template": {},
+  "version": 1,
+  "_meta": {
+    "description": "default settings for common Security Onion Detections indices"
+  }
+}

salt/idstools/soc_idstools.yaml

@@ -8,7 +8,7 @@ idstools:
       global: True
       helpLink: rules.html
     ruleset:
-      description: Defines the ruleset you want to run. Options are ETOPEN or ETPRO.
+      description: 'Defines the ruleset you want to run. Options are ETOPEN or ETPRO. WARNING! Changing the ruleset will remove all existing Suricata rules of the previous ruleset and their associated overrides. This removal cannot be undone.'
       global: True
       regex:  ETPRO\b|ETOPEN\b
       helpLink: rules.html

salt/manager/init.sls

@@ -1,5 +1,5 @@
 # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
@@ -61,7 +61,7 @@ manager_sbin:
     - user: 939
     - group: 939
     - file_mode: 755
-    - exclude_pat: 
+    - exclude_pat:
       - "*_test.py"
 
 yara_update_scripts:
@@ -103,55 +103,6 @@ rules_dir:
     - group: socore
     - makedirs: True
 
-{%    if STRELKAMERGED.rules.enabled %}
-
-strelkarepos:
-  file.managed:
-    - name: /opt/so/conf/strelka/repos.txt
-    - source: salt://strelka/rules/repos.txt.jinja
-    - template: jinja
-    - defaults:
-        STRELKAREPOS: {{ STRELKAMERGED.rules.repos }}
-    - makedirs: True
-
-strelka-yara-update:
-  {%       if MANAGERMERGED.reposync.enabled and not GLOBALS.airgap %} 
-  cron.present:
-  {%       else %}
-  cron.absent:
-  {%       endif %}
-    - user: socore
-    - name: '/usr/sbin/so-yara-update >> /opt/so/log/yarasync/yara-update.log 2>&1'
-    - identifier: strelka-yara-update
-    - hour: '7'
-    - minute: '1'
-
-strelka-yara-download:
-  {%       if MANAGERMERGED.reposync.enabled and not GLOBALS.airgap %}
-  cron.present:
-  {%       else %}
-  cron.absent:
-  {%       endif %}
-    - user: socore
-    - name: '/usr/sbin/so-yara-download >> /opt/so/log/yarasync/yara-download.log 2>&1'
-    - identifier: strelka-yara-download
-    - hour: '7'
-    - minute: '1'
-
-{%      if not GLOBALS.airgap %}
-update_yara_rules:
-  cmd.run:
-    - name: /usr/sbin/so-yara-update
-    - onchanges:
-      - file: yara_update_scripts
-
-download_yara_rules:
-  cmd.run:
-    - name: /usr/sbin/so-yara-download
-    - onchanges:
-      - file: yara_update_scripts
-{%      endif %}
-{%     endif %}
 {% else %}
 
 {{sls}}_state_not_allowed:

salt/manager/tools/sbin_jinja/so-yara-download

@@ -1,51 +0,0 @@
-#!/bin/bash
-NOROOT=1
-. /usr/sbin/so-common
-
-{%- set proxy = salt['pillar.get']('manager:proxy') %}
-{%- set noproxy = salt['pillar.get']('manager:no_proxy', '') %}
-
-# Download the rules from the internet
-{%- if proxy %}
-export http_proxy={{ proxy }} 
-export https_proxy={{ proxy }} 
-export no_proxy="{{ noproxy }}" 
-{%- endif %}
-
-repos="/opt/so/conf/strelka/repos.txt"
-output_dir=/nsm/rules/yara
-gh_status=$(curl -s -o /dev/null -w "%{http_code}" https://github.com)
-clone_dir="/tmp"
-if [ "$gh_status" == "200" ]  || [ "$gh_status" == "301" ]; then
-
-  while IFS= read -r repo; do
-    if ! $(echo "$repo" | grep -qE '^#'); then
-      # Remove old repo if existing bc of previous error condition or unexpected disruption
-      repo_name=`echo $repo | awk -F '/' '{print $NF}'`
-      [ -d $output_dir/$repo_name ] && rm -rf $output_dir/$repo_name
-
-      # Clone repo and make appropriate directories for rules
-      git clone $repo $clone_dir/$repo_name
-      echo "Analyzing rules from $clone_dir/$repo_name..."
-      mkdir -p $output_dir/$repo_name
-      # Ensure a copy of the license is available for the rules
-      [ -f $clone_dir/$repo_name/LICENSE ] && cp $clone_dir/$repo_name/LICENSE $output_dir/$repo_name
-
-      # Copy over rules
-      for i in $(find $clone_dir/$repo_name -name "*.yar*"); do
-        rule_name=$(echo $i | awk -F '/' '{print $NF}')
-        cp $i $output_dir/$repo_name
-     done
-     rm -rf $clone_dir/$repo_name
-    fi
-    done < $repos
-
-  echo "Done!"
-
-/usr/sbin/so-yara-update 
-  
-else
-  echo "Server returned $gh_status status code."
-  echo "No connectivity to Github...exiting..."
-  exit 1
-fi

salt/manager/tools/sbin_jinja/so-yara-update

@@ -1,41 +0,0 @@
-#!/bin/bash
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-NOROOT=1
-. /usr/sbin/so-common
-
-echo "Starting to check for yara rule updates at $(date)..."
-
-newcounter=0
-excludedcounter=0
-excluded_rules=({{ EXCLUDEDRULES | join(' ') }})
-
-# Pull down the SO Rules
-SORULEDIR=/nsm/rules/yara
-OUTPUTDIR=/opt/so/saltstack/local/salt/strelka/rules
-
-mkdir -p $OUTPUTDIR
-# remove all rules prior to copy so we can clear out old rules
-rm -f $OUTPUTDIR/*
-
-for i in $(find $SORULEDIR -name "*.yar" -o -name "*.yara"); do
-  rule_name=$(echo $i | awk -F '/' '{print $NF}')
-  if [[ ! "${excluded_rules[*]}" =~ ${rule_name} ]]; then
-    echo "Adding rule: $rule_name..."
-    cp $i $OUTPUTDIR/$rule_name
-    ((newcounter++))
-  else
-    echo "Excluding rule: $rule_name..."
-    ((excludedcounter++))
-  fi
-done
-
-if [ "$newcounter" -gt 0 ] || [ "$excludedcounter" -gt 0 ];then
-  echo "$newcounter rules added."
-  echo "$excludedcounter rule(s) excluded."
-fi
-
-echo "Finished rule updates at $(date)..."

salt/soc/config.sls

@@ -9,9 +9,16 @@
 include:
   - manager.sync_es_users
 
+socdirtest:
+  file.directory:
+    - name: /opt/so/rules/elastalert/rules
+    - user: 939
+    - group: 939
+    - makedirs: True
+
 socdir:
   file.directory:
-    - name: /opt/so/conf/soc
+    - name: /opt/so/conf/soc/fingerprints
     - user: 939
     - group: 939
     - makedirs: True
@@ -57,6 +64,22 @@ socmotd:
     - mode: 600
     - template: jinja
 
+socsigmafinalpipeline:
+  file.managed:
+    - name: /opt/so/conf/soc/sigma_final_pipeline.yaml
+    - source: salt://soc/files/soc/sigma_final_pipeline.yaml
+    - user: 939
+    - group: 939
+    - mode: 600
+
+socsigmasopipeline:
+  file.managed:
+    - name: /opt/so/conf/soc/sigma_so_pipeline.yaml
+    - source: salt://soc/files/soc/sigma_so_pipeline.yaml
+    - user: 939
+    - group: 939
+    - mode: 600
+
 socbanner:
   file.managed:
     - name: /opt/so/conf/soc/banner.md
@@ -114,6 +137,13 @@ socuploaddir:
     - group: 939
     - makedirs: True
 
+socsigmarepo:
+  file.directory:
+    - name: /opt/so/rules
+    - user: 939
+    - group: 939
+    - mode: 775
+
 {% else %}
 
 {{sls}}_state_not_allowed:

salt/soc/defaults.yaml

@@ -64,7 +64,7 @@ soc:
         icon: fa-external-link-alt
         target: _blank
         links:
-          - 'https://{:sublime.url}/messages/{:sublime.message_group_id}' 
+          - 'https://{:sublime.url}/messages/{:sublime.message_group_id}'
       - name: actionProcessAncestors
         description: actionProcessAncestorsHelp
         icon: fa-people-roof
@@ -1008,6 +1008,13 @@ soc:
           jobDir: jobs
         kratos:
           hostUrl:
+        elastalertengine:
+          communityRulesImportFrequencySeconds: 180
+          elastAlertRulesFolder: /opt/sensoroni/elastalert
+          rulesFingerprintFile: /opt/sensoroni/fingerprints/sigma.fingerprint
+          sigmaRulePackages:
+            - core
+            - emerging_threats_addon
         elastic:
           hostUrl:
           remoteHostUrls: []
@@ -1049,6 +1056,15 @@ soc:
             - rbac/custom_roles
           userFiles:
             - rbac/users_roles
+        strelkaengine:
+          compileYaraPythonScriptPath: /opt/so/conf/strelka/compile_yara.py
+          reposFolder: /opt/sensoroni/yara/repos
+          rulesRepos:
+          - https://github.com/Security-Onion-Solutions/securityonion-yara
+          yaraRulesFolder: /opt/sensoroni/yara/rules
+        suricataengine:
+          communityRulesFile: /nsm/rules/suricata/emerging-all.rules
+          rulesFingerprintFile: /opt/sensoroni/fingerprints/emerging-all.fingerprint
       client:
         enableReverseLookup: false
         docsUrl: /docs/
@@ -1754,3 +1770,55 @@ soc:
                   - amber+strict
                   - red
               customEnabled: false
+        detections:
+          viewEnabled: true
+          createLink: /detection/create
+          eventFetchLimit: 500
+          eventItemsPerPage: 50
+          groupFetchLimit: 50
+          mostRecentlyUsedLimit: 5
+          safeStringMaxLength: 100
+          queryBaseFilter: '_index:"*:so-detection" AND so_kind:detection'
+          eventFields:
+            default:
+              - so_detection.title
+              - so_detection.isEnabled
+              - so_detection.language
+              - so_detection.severity
+          queries:
+            - name: "All Detections"
+              query: "_id:*"
+            - name: "Custom Detections"
+              query: "so_detection.isCommunity:false"
+            - name: "All Detections - Enabled"
+              query: "so_detection.isEnabled:true"
+            - name: "All Detections - Disabled"
+              query: "so_detection.isEnabled:false"
+            - name: "Detection Type - Suricata (NIDS)"
+              query: "so_detection.language:suricata"
+            - name: "Detection Type - Sigma - All"
+              query: "so_detection.language:sigma"
+            - name: "Detection Type - Sigma - Windows"
+              query: 'so_detection.language:sigma AND so_detection.content: "*product: windows*"'
+            - name: "Detection Type - Yara (Strelka)"
+              query: "so_detection.language:yara"
+        detection:
+          presets:
+            severity:
+                customEnabled: false
+                labels:
+                  - unknown
+                  - informational
+                  - low
+                  - medium
+                  - high
+                  - critical
+            language:
+                customEnabled: false
+                labels:
+                  - suricata
+                  - sigma
+                  - yara
+          severityTranslations:
+            minor: low
+            major: high

salt/soc/enabled.sls

@@ -22,12 +22,18 @@ so-soc:
       - sobridge:
         - ipv4_address: {{ DOCKER.containers['so-soc'].ip }}
     - binds:
+      - /nsm/rules:/nsm/rules:rw #Need to tighten this up?
+      - /opt/so/conf/strelka:/opt/sensoroni/yara:rw
+      - /opt/so/rules/elastalert/rules:/opt/sensoroni/elastalert:rw
+      - /opt/so/conf/soc/fingerprints:/opt/sensoroni/fingerprints:rw
       - /nsm/soc/jobs:/opt/sensoroni/jobs:rw
       - /nsm/soc/uploads:/nsm/soc/uploads:rw
       - /opt/so/log/soc/:/opt/sensoroni/logs/:rw
       - /opt/so/conf/soc/soc.json:/opt/sensoroni/sensoroni.json:ro
       - /opt/so/conf/soc/motd.md:/opt/sensoroni/html/motd.md:ro
       - /opt/so/conf/soc/banner.md:/opt/sensoroni/html/login/banner.md:ro
+      - /opt/so/conf/soc/sigma_so_pipeline.yaml:/opt/sensoroni/sigma_so_pipeline.yaml:ro
+      - /opt/so/conf/soc/sigma_final_pipeline.yaml:/opt/sensoroni/sigma_final_pipeline.yaml:rw
       - /opt/so/conf/soc/custom.js:/opt/sensoroni/html/js/custom.js:ro
       - /opt/so/conf/soc/custom_roles:/opt/sensoroni/rbac/custom_roles:ro
       - /opt/so/conf/soc/soc_users_roles:/opt/sensoroni/rbac/users_roles:rw

salt/soc/files/bin/compile_yara.py

@@ -0,0 +1,14 @@
+import os
+import yara
+import glob
+import sys
+
+def compile_yara_rules(rules_dir: str) -> None:
+		compiled_rules_path: str = os.path.join(rules_dir, "rules.yar.compiled")
+		rule_files: list[str] = glob.glob(os.path.join(rules_dir, '**/*.yar'), recursive=True)
+
+		if rule_files:
+				rules: yara.Rules = yara.compile(filepaths={os.path.basename(f): f for f in rule_files})
+				rules.save(compiled_rules_path)
+
+compile_yara_rules(sys.argv[1])

salt/soc/files/soc/sigma_final_pipeline.yaml

@@ -0,0 +1,7 @@
+name: Security Onion - Final Pipeline
+priority: 95
+transformations:
+    - id: override_field_name_mapping
+      type: field_name_mapping
+      mapping:
+        FieldNameToOverride: NewFieldName

salt/soc/files/soc/sigma_so_pipeline.yaml

@@ -0,0 +1,18 @@
+name: Security Onion Baseline Pipeline
+priority: 90
+transformations:
+    - id: baseline_field_name_mapping
+      type: field_name_mapping
+      mapping:
+        cs-method: http.method
+        c-uri: http.uri
+        c-useragent: http.useragent
+        cs-version: http.version
+        uid: user.uid
+        sid: rule.uuid
+        answer: answers
+        query: dns.query.name
+        src_ip: destination.ip.keyword
+        src_port: source.port
+        dst_ip: destination.ip.keyword
+        dst_port: destination.port

salt/soc/soc_soc.yaml

@@ -32,6 +32,14 @@ soc:
         global: True
         advanced: True
         helpLink: soc-customization.html
+      sigma_final_pipeline__yaml:
+        title: Final Sigma Pipeline
+        description: Final Processing Pipeline for Sigma Rules
+        syntax: yaml
+        file: True
+        global: True
+        advanced: True
+        helpLink: soc-customization.html
   config:
     licenseKey:
       title: License Key
@@ -62,6 +70,11 @@ soc:
         global: True
         advanced: True
       modules:
+        elastalertengine:
+          sigmaRulePackages:
+            description: 'Defines the Sigma Community Ruleset you want to run. One of these (core | core+ | core++ | all ) as well as an optional Add-on (emerging_threats_addon). WARNING! Changing the ruleset will remove all existing Sigma rules of the previous ruleset and their associated overrides. This removal cannot be undone.'
+            global: True
+            advanced: False
         elastic:
           index:
             description: Comma-separated list of indices or index patterns (wildcard "*" supported) that SOC will search for records.

salt/strelka/backend/config.sls

@@ -50,16 +50,6 @@ backend_taste:
     - user: 939
     - group: 939
 
-{%   if STRELKAMERGED.rules.enabled %}
-strelkarules:
-  file.recurse:
-    - name: /opt/so/conf/strelka/rules
-    - source: salt://strelka/rules
-    - user: 939
-    - group: 939
-    - clean: True
-{%   endif %}
-
 {% else %}
 
 {{sls}}_state_not_allowed:

salt/strelka/backend/enabled.sls

@@ -42,8 +42,8 @@ strelka_backend:
       {% endfor %}
     {% endif %}
     - restart_policy: on-failure
-    - watch:
-      - file: strelkarules
+    #- watch:
+    #  - file: strelkarules
 
 delete_so-strelka-backend_so-status.disabled:
   file.uncomment:

salt/strelka/config.sls

@@ -1,5 +1,5 @@
 # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
@@ -21,6 +21,13 @@ strelkarulesdir:
     - group: 939
     - makedirs: True
 
+strelkareposdir:
+  file.directory:
+    - name: /opt/so/conf/strelka/repos
+    - user: 939
+    - group: 939
+    - makedirs: True
+
 strelkadatadir:
    file.directory:
     - name: /nsm/strelka