Merge branch '2.4/dev' into jertel/wf

salt/elastalert/files/modules/so/playbook-es.py

@@ -1,38 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-
-from time import gmtime, strftime
-import requests,json
-from elastalert.alerts import Alerter
-
-import urllib3
-urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
-
-class PlaybookESAlerter(Alerter):
-    """
-    Use matched data to create alerts in elasticsearch
-    """
-
-    required_options = set(['play_title','play_url','sigma_level'])
-
-    def alert(self, matches):
-       for match in matches:
-            today = strftime("%Y.%m.%d", gmtime())
-            timestamp = strftime("%Y-%m-%d"'T'"%H:%M:%S"'.000Z', gmtime())
-            headers = {"Content-Type": "application/json"}
-
-            creds = None
-            if 'es_username' in self.rule and 'es_password' in self.rule:
-                creds = (self.rule['es_username'], self.rule['es_password'])
-
-            payload = {"tags":"alert","rule": { "name": self.rule['play_title'],"case_template": self.rule['play_id'],"uuid": self.rule['play_id'],"category": self.rule['rule.category']},"event":{ "severity": self.rule['event.severity'],"module": self.rule['event.module'],"dataset": self.rule['event.dataset'],"severity_label": self.rule['sigma_level']},"kibana_pivot": self.rule['kibana_pivot'],"soc_pivot": self.rule['soc_pivot'],"play_url": self.rule['play_url'],"sigma_level": self.rule['sigma_level'],"event_data": match, "@timestamp": timestamp}
-            url = f"https://{self.rule['es_host']}:{self.rule['es_port']}/logs-playbook.alerts-so/_doc/"
-            requests.post(url, data=json.dumps(payload), headers=headers, verify=False, auth=creds)
-                            
-    def get_info(self):
-        return {'type': 'PlaybookESAlerter'} 

salt/elastalert/files/modules/so/securityonion-es.py

@@ -0,0 +1,63 @@
+# -*- coding: utf-8 -*-
+
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+
+from time import gmtime, strftime
+import requests,json
+from elastalert.alerts import Alerter
+
+import urllib3
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+class SecurityOnionESAlerter(Alerter):
+    """
+    Use matched data to create alerts in Elasticsearch.
+    """
+
+    required_options = set(['detection_title', 'sigma_level'])
+    optional_fields = ['sigma_category', 'sigma_product', 'sigma_service']
+
+    def alert(self, matches):
+        for match in matches:
+            timestamp = strftime("%Y-%m-%d"'T'"%H:%M:%S"'.000Z', gmtime())
+            headers = {"Content-Type": "application/json"}
+
+            creds = None
+            if 'es_username' in self.rule and 'es_password' in self.rule:
+                creds = (self.rule['es_username'], self.rule['es_password'])
+
+            # Start building the rule dict
+            rule_info = {
+                "name": self.rule['detection_title'],
+                "uuid": self.rule['detection_public_id']
+            }
+
+            # Add optional fields if they are present in the rule
+            for field in self.optional_fields:
+                rule_key = field.split('_')[-1]  # Assumes field format "sigma_<key>"
+                if field in self.rule:
+                    rule_info[rule_key] = self.rule[field]
+
+            # Construct the payload with the conditional rule_info
+            payload = {
+                "tags": "alert",
+                "rule": rule_info,
+                "event": {
+                    "severity": self.rule['event.severity'],
+                    "module": self.rule['event.module'],
+                    "dataset": self.rule['event.dataset'],
+                    "severity_label": self.rule['sigma_level']
+                },
+                "sigma_level": self.rule['sigma_level'],
+                "event_data": match,
+                "@timestamp": timestamp
+            }
+            url = f"https://{self.rule['es_host']}:{self.rule['es_port']}/logs-playbook.alerts-so/_doc/"
+            requests.post(url, data=json.dumps(payload), headers=headers, verify=False, auth=creds)
+
+    def get_info(self):
+        return {'type': 'SecurityOnionESAlerter'}

salt/elasticsearch/files/ingest/.fleet_final_pipeline-1

@@ -80,7 +80,7 @@
     { "set":        { "if": "ctx.network?.type == 'ipv6'", "override": true, "field": "destination.ipv6", "value": "true" } },
     { "set":        { "if": "ctx.tags.0 == 'import'", "override": true, "field": "data_stream.dataset", "value": "import" } },
     { "set":        { "if": "ctx.tags.0 == 'import'", "override": true, "field": "data_stream.namespace", "value": "so" } },
-    { "date":       { "if": "ctx.event?.module == 'system'", "field": "event.created", "target_field": "@timestamp", "formats": ["yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'"] } },
+    { "date":       { "if": "ctx.event?.module == 'system'", "field": "event.created", "target_field": "@timestamp","ignore_failure": true, "formats": ["yyyy-MM-dd'T'HH:mm:ss.SSSX","yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'"] } },
     { "community_id":{ "if": "ctx.event?.dataset == 'endpoint.events.network'", "ignore_failure":true }  },
     { "set":        { "if": "ctx.event?.module == 'fim'", "override": true, "field": "event.module", "value": "file_integrity" } },
     { "rename":        { "if": "ctx.winlog?.provider_name == 'Microsoft-Windows-Windows Defender'", "ignore_missing": true, "field": "winlog.event_data.Threat Name", "target_field": "winlog.event_data.threat_name" } },

salt/elasticsearch/soc_elasticsearch.yaml

@@ -121,7 +121,7 @@ elasticsearch:
           cold:
             min_age:
               description: Minimum age of index. ex. 30d - This determines when the index should be moved to the cold tier.  While still searchable, this tier is typically optimized for lower storage costs rather than search speed.
-              regex: ^\[0-9\]{1,5}d$
+              regex: ^[0-9]{1,5}d$
               forcedType: string
               global: True
               helpLink: elasticsearch.html
@@ -134,7 +134,7 @@ elasticsearch:
           warm:
             min_age: 
               description: Minimum age of index. ex. 30d - This determines when the index should be moved to the warm tier. Nodes in the warm tier generally don’t need to be as fast as those in the hot tier.
-              regex: ^\[0-9\]{1,5}d$
+              regex: ^[0-9]{1,5}d$
               forcedType: string
               global: True
             actions:
@@ -147,7 +147,7 @@ elasticsearch:
           delete:
             min_age:
               description: Minimum age of index. ex. 90d - This determines when the index should be deleted.
-              regex: ^\[0-9\]{1,5}d$
+              regex: ^[0-9]{1,5}d$
               forcedType: string
               global: True
               helpLink: elasticsearch.html
@@ -276,7 +276,7 @@ elasticsearch:
           warm:
             min_age:
               description: Minimum age of index. ex. 30d - This determines when the index should be moved to the warm tier. Nodes in the warm tier generally don’t need to be as fast as those in the hot tier.
-              regex: ^\[0-9\]{1,5}d$
+              regex: ^[0-9]{1,5}d$
               forcedType: string
               global: True
               advanced: True
@@ -303,7 +303,7 @@ elasticsearch:
           cold:
             min_age:
               description: Minimum age of index. ex. 30d - This determines when the index should be moved to the cold tier.  While still searchable, this tier is typically optimized for lower storage costs rather than search speed.
-              regex: ^\[0-9\]{1,5}d$
+              regex: ^[0-9]{1,5}d$
               forcedType: string
               global: True
               advanced: True
@@ -319,7 +319,7 @@ elasticsearch:
           delete:
             min_age:
               description: Minimum age of index. This determines when the index should be deleted.
-              regex: ^\[0-9\]{1,5}d$
+              regex: ^[0-9]{1,5}d$
               forcedType: string
               global: True
               advanced: True

salt/soc/defaults.yaml

@@ -1270,6 +1270,7 @@ soc:
           - repo: https://github.com/Security-Onion-Solutions/securityonion-resources
             license: Elastic-2.0
             folder: sigma/stable
+            community: true
           sigmaRulePackages:
             - core
             - emerging_threats_addon
@@ -1327,6 +1328,7 @@ soc:
           rulesRepos:
           - repo: https://github.com/Security-Onion-Solutions/securityonion-yara
             license: DRL
+            community: true
           yaraRulesFolder: /opt/sensoroni/yara/rules
           stateFilePath: /opt/sensoroni/fingerprints/strelkaengine.state
         suricataengine:
@@ -1961,6 +1963,7 @@ soc:
               - rule.name
               - event.severity_label
               - event_data.event.dataset
+              - rule.category
               - event_data.source.ip
               - event_data.source.port
               - event_data.destination.host

setup/so-functions

@@ -1330,6 +1330,10 @@ create_global() {
 	echo "  influxdb_host: '$HOSTNAME'" >> $global_pillar_file
 	echo "  registry_host: '$HOSTNAME'" >> $global_pillar_file
 	echo "  endgamehost: '$ENDGAMEHOST'" >> $global_pillar_file
+
+	if [ "$install_type" = 'EVAL' ]; then
+		echo "  pcapengine: SURICATA" >> $global_pillar_file
+	fi
 }
 
 create_sensoroni_pillar() {