Merge remote-tracking branch 'origin/2.4/dev' into vlb2

salt/elasticsearch/files/ingest/global@custom

@@ -24,6 +24,8 @@
     { "set":        { "if": "ctx.event?.module == 'fim'", "override": true, "field": "event.module", "value": "file_integrity" } },
     { "rename":       { "if": "ctx.winlog?.provider_name == 'Microsoft-Windows-Windows Defender'", "ignore_missing": true, "field": "winlog.event_data.Threat Name", "target_field": "winlog.event_data.threat_name" } },
     { "set":        { "if": "ctx?.metadata?.kafka != null" , "field": "kafka.id", "value": "{{metadata.kafka.partition}}{{metadata.kafka.offset}}{{metadata.kafka.timestamp}}", "ignore_failure": true } },
+    {"append":      {"field":"related.ip","value":["{{source.ip}}","{{destination.ip}}"],"allow_duplicates":false,"if":"ctx?.event?.dataset == 'endpoint.events.network' && ctx?.source?.ip != null","ignore_failure":true}},
+    {"foreach":     {"field":"host.ip","processor":{"append":{"field":"related.ip","value":"{{_ingest._value}}","allow_duplicates":false}},"if":"ctx?.event?.module == 'endpoint'","description":"Extract IPs from Elastic Agent events (host.ip) and adds them to related.ip"}},
     { "remove":     { "field": [ "message2", "type", "fields", "category", "module", "dataset", "event.dataset_temp", "dataset_tag_temp", "module_temp", "datastream_dataset_temp" ], "ignore_missing": true, "ignore_failure": true } }
   ]
 }

salt/elasticsearch/files/ingest/suricata.alert

@@ -9,6 +9,7 @@
     { "rename":{ "field": "rule.signature_id",     "target_field": "rule.uuid", "ignore_failure": true  } },
     { "rename":{ "field": "rule.signature_id",     "target_field": "rule.signature", "ignore_failure": true  } },
     { "rename":{ "field": "message2.payload_printable",     "target_field": "network.data.decoded", "ignore_failure": true  } },
+    { "dissect": { "field": "rule.rule", "pattern": "%{?prefix}content:\"%{dns.query_name}\"%{?remainder}", "ignore_missing": true, "ignore_failure": true } },
     { "pipeline": { "name": "common.nids" } }
   ]
 }

salt/elasticsearch/files/ingest/suricata.common

@@ -18,6 +18,13 @@
     { "set":    { "field": "event.ingested",            "value": "{{@timestamp}}"                                       } },
     { "date": { "field": "message2.timestamp", "target_field": "@timestamp", "formats": ["ISO8601", "UNIX"], "timezone": "UTC", "ignore_failure": true } },
     { "remove":{ "field": "agent",                                                              "ignore_failure": true  } },
+    {"append":{"field":"related.ip","value":["{{source.ip}}","{{destination.ip}}"],"allow_duplicates":false,"ignore_failure":true}},
+    {
+      "script": {
+        "source": "boolean isPrivate(def ip) { if (ip == null) return false; int dot1 = ip.indexOf('.'); if (dot1 == -1) return false; int dot2 = ip.indexOf('.', dot1 + 1); if (dot2 == -1) return false; int first = Integer.parseInt(ip.substring(0, dot1)); if (first == 10) return true; if (first == 192 && ip.startsWith('168.', dot1 + 1)) return true; if (first == 172) { int second = Integer.parseInt(ip.substring(dot1 + 1, dot2)); return second >= 16 && second <= 31; } return false; } String[] fields = new String[] {\"source\", \"destination\"}; for (int i = 0; i < fields.length; i++) { def field = fields[i]; def ip = ctx[field]?.ip; if (ip != null) { if (ctx.network == null) ctx.network = new HashMap(); if (isPrivate(ip)) { if (ctx.network.private_ip == null) ctx.network.private_ip = new ArrayList(); if (!ctx.network.private_ip.contains(ip)) ctx.network.private_ip.add(ip); } else { if (ctx.network.public_ip == null) ctx.network.public_ip = new ArrayList(); if (!ctx.network.public_ip.contains(ip)) ctx.network.public_ip.add(ip); } } }",
+        "ignore_failure": false
+      }
+    },
     { "pipeline": { "if": "ctx?.event?.dataset != null", "name": "suricata.{{event.dataset}}" } }
   ]
 }

salt/soc/config.sls

@@ -52,6 +52,13 @@ socsaltdir:
     - mode: 770
     - makedirs: True
 
+socplaybooksdir:
+  file.directory:
+    - name: /opt/so/conf/soc/playbooks
+    - user: 939
+    - group: 939
+    - makedirs: True
+
 socanalytics:
   file.managed:
     - name: /opt/so/conf/soc/analytics.js

salt/soc/defaults.yaml

@@ -1415,17 +1415,21 @@ soc:
                 license: Elastic-2.0
                 folder: sigma/stable
                 community: true
+                rulesetName: securityonion-resources
               - repo: file:///nsm/rules/custom-local-repos/local-sigma
                 license: Elastic-2.0
                 community: false
+                rulesetName: local-sigma
             airgap:
               - repo: file:///nsm/rules/detect-sigma/repos/securityonion-resources
                 license: Elastic-2.0
                 folder: sigma/stable
                 community: true
+                rulesetName: securityonion-resources
               - repo: file:///nsm/rules/custom-local-repos/local-sigma
                 license: Elastic-2.0
                 community: false
+                rulesetName: local-sigma
           sigmaRulePackages:
             - core
             - emerging_threats_addon
@@ -1500,16 +1504,20 @@ soc:
               - repo: https://github.com/Security-Onion-Solutions/securityonion-yara
                 license: DRL
                 community: true
+                rulesetName: securityonion-yara
               - repo: file:///nsm/rules/custom-local-repos/local-yara
                 license: Elastic-2.0
                 community: false
+                rulesetName: local-yara
             airgap:
               - repo: file:///nsm/rules/detect-yara/repos/securityonion-yara
                 license: DRL
                 community: true
+                rulesetName: securityonion-yara
               - repo: file:///nsm/rules/custom-local-repos/local-yara
                 license: Elastic-2.0
                 community: false
+                rulesetName: local-yara
           yaraRulesFolder: /opt/sensoroni/yara/rules
           stateFilePath: /opt/sensoroni/fingerprints/strelkaengine.state
           integrityCheckFrequencySeconds: 1200

salt/soc/files/soc/sigma_so_pipeline.yaml

@@ -1,45 +1,6 @@
 name: Security Onion Baseline Pipeline
 priority: 90
 transformations:
-vars:
-  document_id:
-   - '{soc_id}'
-  hostname:
-   - '{event_data.host.name}'
-  ProcessGuid:
-   - '{event_data.process.entity_id}'
-  User:
-   - '{user.name}'
-  private_ip:
-   - '{network.private.ip}' 
-  public_ip:
-   - '{network.public.ip}'   
-  related_ip:
-   - '{event_data.related.ip}'
-  related.hosts:
-   - '{event_data.related.hosts'
-  CurrentDirectory:
-   - '{event_data.process.working_directory}'
-  ParentProcessGuid:
-   - '{ParentProcessGuid}'
-  Image:
-   - '{process.executable}'
-  community_id:
-   - '{network.community_id}'
-transformations:
-    - type: value_placeholders
-      include:
-        - 'community_id'
-        - 'document_id'
-        - 'ProcessGuid'      
-        - 'hostname'
-        - 'User'
-        - 'CurrentDirectory'
-        - 'ParentProcessGuid'
-        - 'Image'
-        - 'related_ip'
-        - 'private_ip'
-        - 'public_ip'
     - id: baseline_field_name_mapping
       type: field_name_mapping
       mapping:
@@ -64,14 +25,17 @@ transformations:
         CommandLine: process.command_line
         CurrentDirectory: process.working_directory
         ParentProcessGuid: process.parent.entity_id
-        ParentProcessId: process.parent.pid"
+        ParentProcessId: process.parent.pid
         ParentImage: process.parent.executable
         ParentCommandLine: process.parent.command_line
+        User: user.name
      ## End Temp Linux Mappings ##
+        document_id: _id
         rule.type: event.module
         related_ip: related.ip
         community_id: network.community_id
         event_dataset: event.dataset
+        hostname: host.name
     # Maps "opencanary" product to SO IDH logs
     - id: opencanary_idh_add-fields
       type: add_condition
@@ -181,7 +145,15 @@ transformations:
       rule_conditions:
       - type: logsource
         category: file_event
-        category: file_event
+    # Maps network rules to all network logs
+    # This targets all network logs, all services, generated from endpoints and network
+    - id: network_add-fields
+      type: add_condition
+      conditions:
+        event.category: 'network'
+      rule_conditions:
+      - type: logsource
+        category: network
     # Maps network_connection rules to endpoint network creation logs
     # This is an OS-agnostic mapping, to account for logs that don't specify source OS
     - id: endpoint_network_connection_add-fields
@@ -218,3 +190,41 @@ transformations:
       - type: logsource
         category: network
         service: dns
+    # Maps "network + file" to SO file logs
+    - id: network_file_so_add-fields
+      type: add_condition
+      conditions:
+        event.category: 'network'
+        tags: 'file'
+      rule_conditions:
+      - type: logsource
+        category: network
+        service: file
+    # Maps "network + x509" to SO x509 logs
+    - id: network_x509_so_add-fields
+      type: add_condition
+      conditions:
+        event.category: 'network'
+        tags: 'x509'
+      rule_conditions:
+      - type: logsource
+        category: network
+        service: x509
+    # Maps "network + ssl" to SO ssl logs
+    - id: network_ssl_so_add-fields
+      type: add_condition
+      conditions:
+        event.category: 'network'
+        tags: 'ssl'
+      rule_conditions:
+      - type: logsource
+        category: network
+        service: ssl
+    # Maps file to host or network file events
+    - id: file_so_add-fields
+      type: add_condition
+      conditions:
+        tags: '*file'
+      rule_conditions:
+      - type: logsource
+        category: file

salt/soc/soc_soc.yaml

@@ -344,6 +344,23 @@ soc:
               advanced: True
               forcedType: "[]{}"
               helpLink: sigma.html
+              syntax: json
+              uiElements:
+              - field: rulesetName
+                label: Ruleset Name
+              - field: repo
+                label: Repo URL
+                required: True
+              - field: branch
+                label: Branch
+              - field: license
+                label: License
+                required: True
+              - field: folder
+                label: Folder
+              - field: community
+                label: Community
+                forcedType: bool
             airgap: *eerulesRepos
           sigmaRulePackages:
             description: 'Defines the Sigma Community Ruleset you want to run. One of these (core | core+ | core++ | all ) as well as an optional Add-on (emerging_threats_addon). Once you have changed the ruleset here, the new settings will be applied within 15 minutes. At that point, you will need to wait for the scheduled rule update to take place (by default, every 24 hours), or you can force the update by nagivating to Detections -->  Options dropdown menu --> Elastalert --> Full Update. WARNING! Changing the ruleset will remove all existing non-overlapping Sigma rules of the previous ruleset and their associated overrides. This removal cannot be undone.'
@@ -459,6 +476,23 @@ soc:
               advanced: True
               forcedType: "[]{}"
               helpLink: yara.html
+              syntax: json
+              uiElements:
+              - field: rulesetName
+                label: Ruleset Name
+              - field: repo
+                label: Repo URL
+                required: True
+              - field: branch
+                label: Branch
+              - field: license
+                label: License
+                required: True
+              - field: folder
+                label: Folder
+              - field: community
+                label: Community
+                forcedType: bool
             airgap: *serulesRepos
         suricataengine:
           aiRepoUrl:
@@ -592,7 +626,7 @@ soc:
               label: Query
               required: True
             - field: showSubtitle
-              label: Show Query in Dropdown. 
+              label: Show Query in Dropdown.
               forcedType: bool
           queryToggleFilters:
             description: Customize togglable query filters that apply to all queries. Exclusive toggles will invert the filter if toggled off rather than omitting the filter from the query.

salt/zeek/defaults.yaml

@@ -57,6 +57,7 @@ zeek:
         - cve-2020-0601
         - securityonion/bpfconf
         - securityonion/file-extraction
+        - securityonion/community-id-extended
         - oui-logging
         - icsnpp-modbus
         - icsnpp-dnp3

salt/zeek/policy/securityonion/community-id-extended.zeek

@@ -0,0 +1,40 @@
+##! Extends community ID logging to Files, and SSL by copying
+##! the community_id from the parent connection.
+##!
+##! Note: Requires that protocols/conn/community-id-logging is loaded
+
+module CommunityIDExt;
+
+@load base/protocols/ssl
+@load protocols/conn/community-id-logging
+
+export {
+    redef record SSL::Info += {
+        community_id: string &optional &log;
+    };
+
+    redef record Files::Info += {
+        community_id: string &optional &log;
+    };
+}
+
+# Files
+event file_new(f: fa_file) {
+    if ( f?$conns ) {
+        # Take community_id from first connection that has it
+        for ( cid in f$conns ) {
+            local c = f$conns[cid];
+            if ( c?$conn && c$conn?$community_id ) {
+                f$info$community_id = c$conn$community_id;
+                break;
+            }
+        }
+    }
+}
+
+# SSL Connections
+event ssl_established(c: connection) {
+    if ( c?$conn && c$conn?$community_id && c?$ssl ) {
+        c$ssl$community_id = c$conn$community_id;
+    }
+}