Add Parsing for Playbooks

salt/elasticsearch/files/ingest/global@custom

@@ -22,8 +22,10 @@
     { "date":       { "if": "ctx.event?.module == 'system'", "field": "event.created", "target_field": "@timestamp","ignore_failure": true, "formats": ["yyyy-MM-dd'T'HH:mm:ss.SSSX","yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'"] } },
     { "community_id":{ "if": "ctx.event?.dataset == 'endpoint.events.network'", "ignore_failure":true }  },
     { "set":        { "if": "ctx.event?.module == 'fim'", "override": true, "field": "event.module", "value": "file_integrity" } },
-    { "rename":        { "if": "ctx.winlog?.provider_name == 'Microsoft-Windows-Windows Defender'", "ignore_missing": true, "field": "winlog.event_data.Threat Name", "target_field": "winlog.event_data.threat_name" } },
+    { "rename":     { "if": "ctx.winlog?.provider_name == 'Microsoft-Windows-Windows Defender'", "ignore_missing": true, "field": "winlog.event_data.Threat Name", "target_field": "winlog.event_data.threat_name" } },
     { "set":        { "if": "ctx?.metadata?.kafka != null" , "field": "kafka.id", "value": "{{metadata.kafka.partition}}{{metadata.kafka.offset}}{{metadata.kafka.timestamp}}", "ignore_failure": true } },
+    {"append":      {"field":"related.ip","value":["{{source.ip}}","{{destination.ip}}"],"allow_duplicates":false,"if":"ctx?.event?.dataset == 'endpoint.events.network'","ignore_failure":true}},
+    {"foreach":     {"field":"host.ip","processor":{"append":{"field":"related.ip","value":"{{_ingest._value}}","allow_duplicates":false}},"ignore_failure":true}},
     { "remove":     { "field": [ "message2", "type", "fields", "category", "module", "dataset", "event.dataset_temp", "dataset_tag_temp", "module_temp", "datastream_dataset_temp" ], "ignore_missing": true, "ignore_failure": true } }
   ]
 }

salt/elasticsearch/files/ingest/suricata.alert

@@ -9,6 +9,7 @@
     { "rename":{ "field": "rule.signature_id",     "target_field": "rule.uuid", "ignore_failure": true  } },
     { "rename":{ "field": "rule.signature_id",     "target_field": "rule.signature", "ignore_failure": true  } },
     { "rename":{ "field": "message2.payload_printable",     "target_field": "network.data.decoded", "ignore_failure": true  } },
+    { "dissect": { "field": "rule.rule", "pattern": "%{?prefix}content:\"%{dns.query_name}\"%{?remainder}", "ignore_missing": true, "tag": "suricata" } },
     { "pipeline": { "name": "common.nids" } }
   ]
 }

salt/elasticsearch/files/ingest/suricata.common

@@ -18,6 +18,13 @@
     { "set":    { "field": "event.ingested",            "value": "{{@timestamp}}"                                       } },
     { "date": { "field": "message2.timestamp", "target_field": "@timestamp", "formats": ["ISO8601", "UNIX"], "timezone": "UTC", "ignore_failure": true } },
     { "remove":{ "field": "agent",                                                              "ignore_failure": true  } },
+    {"append":{"field":"related.ip","value":["{{source.ip}}","{{destination.ip}}"],"allow_duplicates":false,"ignore_failure":true}},
+    {
+      "script": {
+        "source": "boolean isPrivate(def ip) {   if (ip == null) return false;   if (ip.startsWith('10.')) return true;   if (ip.startsWith('192.168.')) return true;   if (ip.startsWith('172.')) {     String[] parts = ip.split('\\\\.');     if (parts.length > 1) {       int second = Integer.parseInt(parts[1]);       if (second >= 16 && second <= 31) return true;     }   }   return false; } String[] fields = new String[] {\"source\", \"destination\"}; for (int i = 0; i < fields.length; i++) {   def field = fields[i];   def ip = ctx.containsKey(field) && ctx[field].containsKey('ip') ? ctx[field].ip : null;   if (ip != null) {     if (ctx.network == null) ctx.network = new HashMap();     String netField = isPrivate(ip) ? \"private\" : \"public\";     if (!ctx.network.containsKey(netField)) {       ctx.network[netField] = new HashMap();     }     if (!ctx.network[netField].containsKey(\"ip\")) {       ctx.network[netField].ip = new ArrayList();     }     if (!ctx.network[netField].ip.contains(ip)) {       ctx.network[netField].ip.add(ip);     }   } }",
+        "ignore_failure": true
+      }
+    },
     { "pipeline": { "if": "ctx?.event?.dataset != null", "name": "suricata.{{event.dataset}}" } }
   ]
 }

salt/soc/files/soc/sigma_so_pipeline.yaml

@@ -1,6 +1,45 @@
 name: Security Onion Baseline Pipeline
 priority: 90
 transformations:
+vars:
+  document_id:
+   - '{soc_id}'
+  hostname:
+   - '{event_data.host.name}'
+  ProcessGuid:
+   - '{event_data.process.entity_id}'
+  User:
+   - '{user.name}'
+  private_ip:
+   - '{network.private.ip}' 
+  public_ip:
+   - '{network.public.ip}'   
+  related_ip:
+   - '{event_data.related.ip}'
+  related.hosts:
+   - '{event_data.related.hosts'
+  CurrentDirectory:
+   - '{event_data.process.working_directory}'
+  ParentProcessGuid:
+   - '{ParentProcessGuid}'
+  Image:
+   - '{process.executable}'
+  community_id:
+   - '{network.community_id}'
+transformations:
+    - type: value_placeholders
+      include:
+        - 'community_id'
+        - 'document_id'
+        - 'ProcessGuid'      
+        - 'hostname'
+        - 'User'
+        - 'CurrentDirectory'
+        - 'ParentProcessGuid'
+        - 'Image'
+        - 'related_ip'
+        - 'private_ip'
+        - 'public_ip'
     - id: baseline_field_name_mapping
       type: field_name_mapping
       mapping:
@@ -12,12 +51,27 @@ transformations:
         sid: rule.uuid
         answer: answers
         query: dns.query.name
-        src_ip: source.ip.keyword
+        src_ip: source.ip
         src_port: source.port
-        dst_ip: destination.ip.keyword
+        dst_ip: destination.ip
         dst_port: destination.port
         winlog.event_data.User: user.name
         logtype: event.code # OpenCanary
+      ## Start Temp Linux Mappings ##
+        ProcessGuid: process.entity_id
+        ProcessId: process.pid
+        Image: process.executable
+        CommandLine: process.command_line
+        CurrentDirectory: process.working_directory
+        ParentProcessGuid: process.parent.entity_id
+        ParentProcessId: process.parent.pid"
+        ParentImage: process.parent.executable
+        ParentCommandLine: process.parent.command_line
+     ## End Temp Linux Mappings ##
+        rule.type: event.module
+        related_ip: related.ip
+        community_id: network.community_id
+        event_dataset: event.dataset
     # Maps "opencanary" product to SO IDH logs
     - id: opencanary_idh_add-fields
       type: add_condition
@@ -127,3 +181,40 @@ transformations:
       rule_conditions:
       - type: logsource
         category: file_event
+        category: file_event
+    # Maps network_connection rules to endpoint network creation logs
+    # This is an OS-agnostic mapping, to account for logs that don't specify source OS
+    - id: endpoint_network_connection_add-fields
+      type: add_condition
+      conditions:
+        event.category: 'network'
+        event.type: 'start'
+      rule_conditions:
+      - type: logsource
+        category: network_connection
+    # Maps "alert" category to SO Alert events
+    - id: alert_so_add-fields
+      type: add_condition
+      conditions:
+        tags: 'alert'
+      rule_conditions:
+      - type: logsource
+        category: alert
+    # Maps "network + connection" to SO connection logs
+    - id: network_connection_so_add-fields
+      type: add_condition
+      conditions:
+        tags: 'conn'
+      rule_conditions:
+      - type: logsource
+        category: network
+        service: connection
+    # Maps "network + dns" to SO DNS logs
+    - id: network_dns_so_add-fields
+      type: add_condition
+      conditions:
+        tags: 'dns'
+      rule_conditions:
+      - type: logsource
+        category: network
+        service: dns