Merge pull request #14638 from Security-Onion-Solutions/2.4/playbooks-parsing

Add parsing for Playbook

salt/elasticsearch/files/ingest/global@custom

@@ -24,6 +24,8 @@
     { "set":        { "if": "ctx.event?.module == 'fim'", "override": true, "field": "event.module", "value": "file_integrity" } },
     { "rename":       { "if": "ctx.winlog?.provider_name == 'Microsoft-Windows-Windows Defender'", "ignore_missing": true, "field": "winlog.event_data.Threat Name", "target_field": "winlog.event_data.threat_name" } },
     { "set":        { "if": "ctx?.metadata?.kafka != null" , "field": "kafka.id", "value": "{{metadata.kafka.partition}}{{metadata.kafka.offset}}{{metadata.kafka.timestamp}}", "ignore_failure": true } },
+    {"append":      {"field":"related.ip","value":["{{source.ip}}","{{destination.ip}}"],"allow_duplicates":false,"if":"ctx?.event?.dataset == 'endpoint.events.network'","ignore_failure":true}},
+    {"foreach":     {"field":"host.ip","processor":{"append":{"field":"related.ip","value":"{{_ingest._value}}","allow_duplicates":false}},"ignore_failure":true}},
     { "remove":     { "field": [ "message2", "type", "fields", "category", "module", "dataset", "event.dataset_temp", "dataset_tag_temp", "module_temp", "datastream_dataset_temp" ], "ignore_missing": true, "ignore_failure": true } }
   ]
 }

salt/elasticsearch/files/ingest/suricata.alert

@@ -9,6 +9,7 @@
     { "rename":{ "field": "rule.signature_id",     "target_field": "rule.uuid", "ignore_failure": true  } },
     { "rename":{ "field": "rule.signature_id",     "target_field": "rule.signature", "ignore_failure": true  } },
     { "rename":{ "field": "message2.payload_printable",     "target_field": "network.data.decoded", "ignore_failure": true  } },
+    { "dissect": { "field": "rule.rule", "pattern": "%{?prefix}content:\"%{dns.query_name}\"%{?remainder}", "ignore_missing": true, "ignore_failure": true } },
     { "pipeline": { "name": "common.nids" } }
   ]
 }

salt/elasticsearch/files/ingest/suricata.common

@@ -18,6 +18,13 @@
     { "set":    { "field": "event.ingested",            "value": "{{@timestamp}}"                                       } },
     { "date": { "field": "message2.timestamp", "target_field": "@timestamp", "formats": ["ISO8601", "UNIX"], "timezone": "UTC", "ignore_failure": true } },
     { "remove":{ "field": "agent",                                                              "ignore_failure": true  } },
+    {"append":{"field":"related.ip","value":["{{source.ip}}","{{destination.ip}}"],"allow_duplicates":false,"ignore_failure":true}},
+    {
+      "script": {
+        "source": "boolean isPrivate(def ip) { if (ip == null) return false; int dot1 = ip.indexOf('.'); if (dot1 == -1) return false; int dot2 = ip.indexOf('.', dot1 + 1); if (dot2 == -1) return false; int first = Integer.parseInt(ip.substring(0, dot1)); if (first == 10) return true; if (first == 192 && ip.startsWith('168.', dot1 + 1)) return true; if (first == 172) { int second = Integer.parseInt(ip.substring(dot1 + 1, dot2)); return second >= 16 && second <= 31; } return false; } String[] fields = new String[] {\"source\", \"destination\"}; for (int i = 0; i < fields.length; i++) { def field = fields[i]; def ip = ctx[field]?.ip; if (ip != null) { if (ctx.network == null) ctx.network = new HashMap(); if (isPrivate(ip)) { if (ctx.network.private_ip == null) ctx.network.private_ip = new ArrayList(); if (!ctx.network.private_ip.contains(ip)) ctx.network.private_ip.add(ip); } else { if (ctx.network.public_ip == null) ctx.network.public_ip = new ArrayList(); if (!ctx.network.public_ip.contains(ip)) ctx.network.public_ip.add(ip); } } }",
+        "ignore_failure": false
+      }
+    },
     { "pipeline": { "if": "ctx?.event?.dataset != null", "name": "suricata.{{event.dataset}}" } }
   ]
 }

salt/soc/config.sls

@@ -52,6 +52,13 @@ socsaltdir:
     - mode: 770
     - makedirs: True
 
+socplaybooksdir:
+  file.directory:
+    - name: /opt/so/conf/soc/playbooks
+    - user: 939
+    - group: 939
+    - makedirs: True
+
 socanalytics:
   file.managed:
     - name: /opt/so/conf/soc/analytics.js

salt/soc/files/soc/sigma_so_pipeline.yaml

@@ -1,45 +1,6 @@
 name: Security Onion Baseline Pipeline
 priority: 90
 transformations:
-vars:
-  document_id:
-   - '{soc_id}'
-  hostname:
-   - '{event_data.host.name}'
-  ProcessGuid:
-   - '{event_data.process.entity_id}'
-  User:
-   - '{user.name}'
-  private_ip:
-   - '{network.private.ip}' 
-  public_ip:
-   - '{network.public.ip}'   
-  related_ip:
-   - '{event_data.related.ip}'
-  related.hosts:
-   - '{event_data.related.hosts'
-  CurrentDirectory:
-   - '{event_data.process.working_directory}'
-  ParentProcessGuid:
-   - '{ParentProcessGuid}'
-  Image:
-   - '{process.executable}'
-  community_id:
-   - '{network.community_id}'
-transformations:
-    - type: value_placeholders
-      include:
-        - 'community_id'
-        - 'document_id'
-        - 'ProcessGuid'      
-        - 'hostname'
-        - 'User'
-        - 'CurrentDirectory'
-        - 'ParentProcessGuid'
-        - 'Image'
-        - 'related_ip'
-        - 'private_ip'
-        - 'public_ip'
     - id: baseline_field_name_mapping
       type: field_name_mapping
       mapping:
@@ -64,14 +25,17 @@ transformations:
         CommandLine: process.command_line
         CurrentDirectory: process.working_directory
         ParentProcessGuid: process.parent.entity_id
-        ParentProcessId: process.parent.pid"
+        ParentProcessId: process.parent.pid
         ParentImage: process.parent.executable
         ParentCommandLine: process.parent.command_line
+        User: user.name
      ## End Temp Linux Mappings ##
+        document_id: _id
         rule.type: event.module
         related_ip: related.ip
         community_id: network.community_id
         event_dataset: event.dataset
+        hostname: host.name
     # Maps "opencanary" product to SO IDH logs
     - id: opencanary_idh_add-fields
       type: add_condition
@@ -181,7 +145,15 @@ transformations:
       rule_conditions:
       - type: logsource
         category: file_event
-        category: file_event
+    # Maps network rules to all network logs
+    # This targets all network logs, all services, generated from endpoints and network
+    - id: network_add-fields
+      type: add_condition
+      conditions:
+        event.category: 'network'
+      rule_conditions:
+      - type: logsource
+        category: network
     # Maps network_connection rules to endpoint network creation logs
     # This is an OS-agnostic mapping, to account for logs that don't specify source OS
     - id: endpoint_network_connection_add-fields
@@ -218,3 +190,41 @@ transformations:
       - type: logsource
         category: network
         service: dns
+    # Maps "network + file" to SO file logs
+    - id: network_file_so_add-fields
+      type: add_condition
+      conditions:
+        event.category: 'network'
+        tags: 'file'
+      rule_conditions:
+      - type: logsource
+        category: network
+        service: file
+    # Maps "network + x509" to SO x509 logs
+    - id: network_x509_so_add-fields
+      type: add_condition
+      conditions:
+        event.category: 'network'
+        tags: 'x509'
+      rule_conditions:
+      - type: logsource
+        category: network
+        service: x509
+    # Maps "network + ssl" to SO ssl logs
+    - id: network_ssl_so_add-fields
+      type: add_condition
+      conditions:
+        event.category: 'network'
+        tags: 'ssl'
+      rule_conditions:
+      - type: logsource
+        category: network
+        service: ssl
+    # Maps file to host or network file events
+    - id: file_so_add-fields
+      type: add_condition
+      conditions:
+        tags: '*file'
+      rule_conditions:
+      - type: logsource
+        category: file

salt/zeek/defaults.yaml

@@ -57,6 +57,7 @@ zeek:
         - cve-2020-0601
         - securityonion/bpfconf
         - securityonion/file-extraction
+        - securityonion/community-id-extended
         - oui-logging
         - icsnpp-modbus
         - icsnpp-dnp3

salt/zeek/policy/securityonion/community-id-extended.zeek

@@ -0,0 +1,40 @@
+##! Extends community ID logging to Files, and SSL by copying
+##! the community_id from the parent connection.
+##!
+##! Note: Requires that protocols/conn/community-id-logging is loaded
+
+module CommunityIDExt;
+
+@load base/protocols/ssl
+@load protocols/conn/community-id-logging
+
+export {
+    redef record SSL::Info += {
+        community_id: string &optional &log;
+    };
+
+    redef record Files::Info += {
+        community_id: string &optional &log;
+    };
+}
+
+# Files
+event file_new(f: fa_file) {
+    if ( f?$conns ) {
+        # Take community_id from first connection that has it
+        for ( cid in f$conns ) {
+            local c = f$conns[cid];
+            if ( c?$conn && c$conn?$community_id ) {
+                f$info$community_id = c$conn$community_id;
+                break;
+            }
+        }
+    }
+}
+
+# SSL Connections
+event ssl_established(c: connection) {
+    if ( c?$conn && c$conn?$community_id && c?$ssl ) {
+        c$ssl$community_id = c$conn$community_id;
+    }
+}