Pull in Suricata changes

salt/elasticsearch/files/ingest/suricata.common

@@ -15,6 +15,6 @@
     { "set":         { "field": "ingest.timestamp",        "value": "{{@timestamp}}"  } },
     { "date": { "field": "message2.timestamp", "target_field": "@timestamp", "formats": ["ISO8601", "UNIX"], "timezone": "UTC", "ignore_failure": true } },
     { "remove":{ "field": "agent", 						"ignore_failure": true	} },
-    { "pipeline": { "name": "suricata.{{dataset}}" } }
+    { "pipeline": { "if": "ctx?.dataset != null", "name": "suricata.{{dataset}}" } }
   ]
-}
+}

salt/elasticsearch/files/ingest/suricata.dns

@@ -7,8 +7,13 @@
     { "rename": 	{ "field": "message2.dns.tx_id",		"target_field": "dns.id",	"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.dns.version", 		"target_field": "dns.version",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.dns.rrname", 		"target_field": "dns.query.name",		"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.grouped.A", 		"target_field": "dns.answers",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.dns.flags", 		"target_field": "dns.flags",			"ignore_missing": true 	} },
-
+    { "rename": 	{ "field": "message2.dns.qr", 		"target_field": "dns.qr",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.dns.rd", 		"target_field": "dns.recursion.desired",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.dns.ra", 		"target_field": "dns.recursion.available",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.dns.rcode", 		"target_field": "dns.response.code",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.grouped.A", 		"target_field": "dns.answers.data",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.grouped.CNAME", 		"target_field": "dns.answers.name",			"ignore_missing": true 	} },
     { "pipeline": { "name": "common" } }
   ]
-}
+}

salt/elasticsearch/files/ingest/suricata.fileinfo

@@ -1,8 +1,18 @@
 {
   "description" : "suricata.fileinfo",
   "processors" : [
+    { "set":      { "field": "dataset",        "value": "file"  } },
     { "rename": 	{ "field": "message2.proto", 		"target_field": "network.transport",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.app_proto", 		"target_field": "network.protocol",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.fileinfo.filename", 		"target_field": "file.name",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.fileinfo.gaps", 		"target_field": "file.bytes.missing",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.fileinfo.magic", 		"target_field": "file.mime_type",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.fileinfo.md5", 		"target_field": "hash.md5",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.fileinfo.sha1", 		"target_field": "hash.sha1",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.fileinfo.sid", 		"target_field": "rule.uuid",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.fileinfo.size", 		"target_field": "file.size",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.fileinfo.state", 		"target_field": "file.state",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.fileinfo.stored", 		"target_field": "file.saved",		"ignore_missing": true 	} },
     { "pipeline": { "name": "common" } }
   ]
-}
+}

salt/elasticsearch/files/ingest/suricata.krb5

@@ -1,8 +1,15 @@
 {
   "description" : "suricata.krb5",
   "processors" : [
+    { "set":      { "field": "dataset",        "value": "kerberos"  } },
     { "rename": 	{ "field": "message2.proto", 		"target_field": "network.transport",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.app_proto", 		"target_field": "network.protocol",		"ignore_missing": true 	} },
+    { "rename":         { "field": "message2.krb5.msg_type",                "target_field": "kerberos.request_type",             "ignore_missing": true  } },
+    { "rename":         { "field": "message2.krb5.cname",                "target_field": "kerberos.client",             "ignore_missing": true  } },
+    { "rename":         { "field": "message2.krb5.realm",                "target_field": "kerberos.realm",             "ignore_missing": true  } },
+    { "rename":         { "field": "message2.krb5.sname",                "target_field": "kerberos.service",             "ignore_missing": true  } },
+    { "rename":         { "field": "message2.krb5.encryption",                "target_field": "kerberos.ticket.cipher",             "ignore_missing": true  } },
+    { "rename":         { "field": "message2.krb.weak_encryption",                "target_field": "kerberos.weak_encryption",             "ignore_missing": true  } },
     { "pipeline": { "name": "common" } }
   ]
-}
+}

salt/elasticsearch/files/ingest/suricata.rdp

@@ -3,6 +3,26 @@
   "processors" : [
     { "rename": 	{ "field": "message2.proto", 		"target_field": "network.transport",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.app_proto", 		"target_field": "network.protocol",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.rdp.tx_id", 		"target_field": "rdp.tx_id",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.rdp.event_type", 		"target_field": "rdp.event_type",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.rdp.cookie", 		"target_field": "rdp.cookie",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.rdp.client.version", 		"target_field": "rdp.client_version",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.rdp.client.desktop_width", 		"target_field": "rdp.desktop__width",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.rdp.client.desktop__height", 		"target_field": "rdp.desktop__height",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.rdp.client.color_depth", 		"target_field": "rdp.requested_color_depth",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.rdp.client.keyboard_layout", 		"target_field": "rdp.keyboard_layout",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.rdp.client.build", 		"target_field": "rdp.client_build",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.rdp.client.client_name", 		"target_field": "client.name",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.rdp.client.keyboard_type", 		"target_field": "rdp.keyboard_type",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.rdp.client.function_keys", 		"target_field": "rdp.function_keys",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.rdp.client.product_id", 		"target_field": "rdp.product_id",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.rdp.client.capabilities", 		"target_field": "rdp.client_capabilities",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.rdp.client.id", 		"target_field": "rdp.client_id",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.rdp.channels", 		"target_field": "rdp.channels",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.rdp.server_supports", 		"target_field": "rdp.server_supports",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.rdp.x509_serials", 		"target_field": "rdp.x509_serials",		"ignore_missing": true 	} },
+
+
     { "pipeline": { "name": "common" } }
   ]
-}
+}

salt/elasticsearch/files/ingest/suricata.smb

@@ -3,6 +3,30 @@
   "processors" : [
     { "rename": 	{ "field": "message2.proto", 		"target_field": "network.transport",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.app_proto", 		"target_field": "network.protocol",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.smb.id", 		"target_field": "smb.id",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.smb.dialect", 		"target_field": "smb.dialect",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.smb.command", 		"target_field": "smb.command",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.smb.status", 		"target_field": "smb.status",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.smb.status_code", 		"target_field": "smb.status_code",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.smb.session_id", 		"target_field": "smb.session_id",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.smb.tree_id", 		"target_field": "smb.tree_id",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.smb.filename", 		"target_field": "smb.filename",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.smb.disposition", 		"target_field": "smb.disposition",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.smb.access", 		"target_field": "smb.access",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.smb.created", 		"target_field": "smb.created",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.smb.accessed", 		"target_field": "smb.accessed",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.smb.modified", 		"target_field": "smb.modified ",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.smb.changed", 		"target_field": "smb.changed",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.smb.size", 		"target_field": "smb.size",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.smb.share", 		"target_field": "smb.share",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.smb.share_type", 		"target_field": "smb.share_type",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.client_dialects", 		"target_field": "smb.client_dialects",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.client_guid", 		"target_field": "smb.client_guid",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.server_guid", 		"target_field": "smb.server_guid",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.request.native_us", 		"target_field": "smb.request.native_us",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.request.native_lm", 		"target_field": "smb.request.native_lm",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.response.native_os", 		"target_field": "smb.response.native_os",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.response.native_lm", 		"target_field": "smb.response.native_lm",		"ignore_missing": true 	} },
     { "pipeline": { "name": "common" } }
   ]
-}
+}

salt/elasticsearch/files/ingest/suricata.snmp

@@ -5,6 +5,8 @@
     { "rename": 	{ "field": "message2.app_proto", 		"target_field": "network.protocol",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.snmp.version", 		"target_field": "snmp.version",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.snmp.community", 		"target_field": "snmp.community",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.snmp.pdu_type", 		"target_field": "snmp.pdu_type",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.snmp.vars", 		"target_field": "snmp.vars",		"ignore_missing": true 	} },
     { "pipeline": { "name": "common" } }
   ]
-}
+}

salt/elasticsearch/files/ingest/suricata.tftp

@@ -3,6 +3,10 @@
   "processors" : [
     { "rename": 	{ "field": "message2.proto", 		"target_field": "network.transport",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.app_proto", 		"target_field": "network.protocol",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.tftp.packet", 		"target_field": "tftp.packet",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.tftp.file", 		"target_field": "tftp.file",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.tftp.mode", 		"target_field": "tftp.mode",		"ignore_missing": true 	} },
+    
     { "pipeline": { "name": "common" } }
   ]
-}
+}

salt/elasticsearch/files/ingest/suricata.tls

@@ -1,8 +1,22 @@
 {
   "description" : "suricata.tls",
   "processors" : [
+    { "set":      { "field": "dataset",        "value": "ssl"  } },
     { "rename": 	{ "field": "message2.proto", 		"target_field": "network.transport",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.app_proto", 		"target_field": "network.protocol",		"ignore_missing": true 	} },
+    { "rename":         { "field": "message2.tls.subject",                "target_field": "ssl.certificate.subject",             "ignore_missing": true  } },
+    { "rename":         { "field": "message2.tls.serial",                "target_field": "ssl.certificate.serial",             "ignore_missing": true  } },
+    { "rename":         { "field": "message2.tls.fingerprint",                "target_field": "ssl.certificate.fingerprint",             "ignore_missing": true  } },
+    { "rename":         { "field": "message2.tls.version",                "target_field": "ssl.certificate.version",             "ignore_missing": true  } },
+    { "rename":         { "field": "message2.tls.ja3.hash",                "target_field": "hash.ja3",             "ignore_missing": true  } },
+    { "rename":         { "field": "message2.tls.ja3.hash.string",                "target_field": "hash.ja3_string",             "ignore_missing": true  } },
+    { "rename":         { "field": "message2.tls.ja3s.hash",                "target_field": "hash.ja3s",             "ignore_missing": true  } },
+    { "rename":         { "field": "message2.tls.ja3s.hash.string",                "target_field": "hash.ja3s_string",             "ignore_missing": true  } },
+    { "rename":         { "field": "message2.tls.notbefore",                "target_field": "x509.certificate.not_valid_before",             "ignore_missing": true  } },
+    { "rename":         { "field": "message2.tls.notafter",                "target_field": "x509.certificate.not_valid_after",             "ignore_missing": true  } },
+    { "rename":         { "field": "message2.tls.sni",                "target_field": "ssl.server_name",             "ignore_missing": true  } },
+    { "rename":         { "field": "message2.tls.issuerdn",                "target_field": "ssl.certificate.issuer",             "ignore_missing": true  } },
+    { "rename":         { "field": "message2.tls.session_resumed",                "target_field": "ssl.session_resumed",             "ignore_missing": true  } },
     { "pipeline": { "name": "common" } }
   ]
-}
+}

salt/idstools/etc/rulecat.conf

@@ -3,17 +3,26 @@
 {%- set OINKCODE = salt['pillar.get']('idstools:config:oinkcode', '' ) -%}
 {%- set ISAIRGAP = salt['pillar.get']('global:airgap', 'False') -%}
 {%- set MANAGERIP = salt['pillar.get']('global:managerip', '') -%}
+{%- set ENGINE = salt['pillar.get']('global:mdengine', '') %}
 {%- if ISAIRGAP is sameas true -%}
 --merged=/opt/so/rules/nids/all.rules
 --local=/opt/so/rules/nids/local.rules
+{%- if ENGINE == "SURICATA" %}
+--local=/opt/so/rules/nids/sorules/extraction.rules
+--local=/opt/so/rules/nids/sorules/filters.rules
+{%- endif %}
 --url=http://{{ MANAGERIP }}:7788/rules/emerging-all.rules
 --disable=/opt/so/idstools/etc/disable.conf
 --enable=/opt/so/idstools/etc/enable.conf
 --modify=/opt/so/idstools/etc/modify.conf
 {%- else -%}
---suricata-version=5.0
+--suricata-version=6.0
 --merged=/opt/so/rules/nids/all.rules
 --local=/opt/so/rules/nids/local.rules
+{%- if ENGINE == "SURICATA" %}
+--local=/opt/so/rules/nids/sorules/extraction.rules
+--local=/opt/so/rules/nids/sorules/filters.rules
+{%- endif %}
 --disable=/opt/so/idstools/etc/disable.conf
 --enable=/opt/so/idstools/etc/enable.conf
 --modify=/opt/so/idstools/etc/modify.conf
@@ -29,4 +38,4 @@
 {%- for URL in URLS %}
 --url={{ URL }}
 {%- endfor %}
-{%- endif %}
+{%- endif %}

salt/idstools/init.sls

@@ -18,6 +18,7 @@
 {% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %}
 {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %}
 {% set MANAGER = salt['grains.get']('master') %}
+{% set ENGINE = salt['pillar.get']('global:mdengine', '') %}
 # IDSTools Setup
 idstoolsdir:
   file.directory:
@@ -55,14 +56,12 @@ rulesdir:
     - group: 939
     - makedirs: True
 
-# Don't show changes because all.rules can be large
 synclocalnidsrules:
   file.recurse:
     - name: /opt/so/rules/nids/
     - source: salt://idstools/
     - user: 939
     - group: 939
-    - show_changes: False
     - include_pat: 'E@.rules'
 
 so-idstools:
@@ -87,4 +86,4 @@ append_so-idstools_so-status.conf:
   test.fail_without_changes:
     - name: {{sls}}_state_not_allowed
 
-{% endif%}
+{% endif%}

salt/idstools/sorules/extraction.rules

@@ -0,0 +1,26 @@
+# Extract all PDF mime type
+alert http any any -> any any (msg:"FILE pdf detected"; filemagic:"PDF document"; filestore; sid:1100000; rev:1;)
+alert smtp any any -> any any (msg:"FILE pdf detected"; filemagic:"PDF document"; filestore; sid:1100001; rev:1;)
+alert nfs any any -> any any (msg:"FILE pdf detected"; filemagic:"PDF document"; filestore; sid:1100002; rev:1;)
+alert smb any any -> any any (msg:"FILE pdf detected"; filemagic:"PDF document"; filestore; sid:1100003; rev:1;)
+# Extract EXE/DLL file types
+alert http any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; sid:1100004; rev:1;)
+alert smtp any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; sid:1100005; rev:1;)
+alert nfs any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; sid:1100006; rev:1;)
+alert smb any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; sid:11000007; rev:1;)
+alert http any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; sid:1100008; rev:1;)
+alert smtp any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; sid:1100009; rev:1;)
+alert nfs any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; sid:1100010; rev:1;)
+alert smb any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; sid:11000011; rev:1;)
+
+# Extract all Zip files
+alert http any any -> any any (msg:"FILE ZIP detected"; filemagic:"Zip"; filestore; sid:1100012; rev:1;)
+alert smtp any any -> any any (msg:"FILE EXE detected"; filemagic:"Zip"; filestore; sid:1100013; rev:1;)
+alert nfs any any -> any any (msg:"FILE EXE detected"; filemagic:"Zip"; filestore; sid:1100014; rev:1;)
+alert smb any any -> any any (msg:"FILE EXE detected"; filemagic:"Zip"; filestore; sid:11000015; rev:1;)
+
+# Extract Word Docs
+alert http any any -> any any (msg:"FILE EXE detected"; filemagic:"Composite Document File V2 Document"; filestore; sid:1100016; rev:1;)
+alert smtp any any -> any any (msg:"FILE EXE detected"; filemagic:"Composite Document File V2 Document"; filestore; sid:1100017; rev:1;)
+alert nfs any any -> any any (msg:"FILE EXE detected"; filemagic:"Composite Document File V2 Document"; filestore; sid:1100018; rev:1;)
+alert smb any any -> any any (msg:"FILE EXE detected"; filemagic:"Composite Document File V2 Document"; filestore; sid:1100019; rev:1;)

salt/idstools/sorules/filer.rules

@@ -0,0 +1,3 @@
+# Start the filters at sid 1200000
+# Example of filtering out google.com from being dns logged.
+#config dns any any -> any any (dns.query; content:"google.com"; config: logging disable, type tx, scope tx; sid:1200000;)

salt/strelka/init.sls

@@ -20,6 +20,7 @@
 {% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %}
 {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %}
 {% set STRELKA_RULES = salt['pillar.get']('strelka:rules', '1') %}
+{% set ENGINE = salt['pillar.get']('global:mdengine', '') %}
 
 # Strelka config
 strelkaconfdir:
@@ -188,6 +189,16 @@ strelka_zeek_extracted_sync_old:
     - name: '[ -d /nsm/zeek/extracted/complete/ ] && mv /nsm/zeek/extracted/complete/* /nsm/strelka/ > /dev/null 2>&1'
     - minute: '*'
 
+{% if ENGINE == "SURICATA" %}
+
+strelka_suricata_extracted_sync:
+  cron.present:
+    - user: root
+    - identifier: zeek-extracted-strelka-sync
+    - name: '[ -d /nsm/suricata/extracted/ ] && find /nsm/suricata/extracted/* -not \( -path /nsm/suricata/extracted/tmp -prune \) -type f -print0 | xargs -0 -I {} mv {} /nsm/strelka/unprocessed/ > /dev/null 2>&1'
+    - minute: '*'
+
+{% else %}
 strelka_zeek_extracted_sync:
   cron.present:
     - user: root
@@ -195,10 +206,11 @@ strelka_zeek_extracted_sync:
     - name: '[ -d /nsm/zeek/extracted/complete/ ] && mv /nsm/zeek/extracted/complete/* /nsm/strelka/unprocessed/ > /dev/null 2>&1'
     - minute: '*'
 
+{% endif %}
 {% else %}
 
 {{sls}}_state_not_allowed:
   test.fail_without_changes:
     - name: {{sls}}_state_not_allowed
 
-{% endif %}
+{% endif %}

salt/suricata/defaults.yaml

@@ -156,8 +156,6 @@ suricata:
             mode: extra-data
             deployment: reverse
             header: X-Forwarded-For
-      - file-store:
-          enabled: "no"
       - tcp-data:
           enabled: "no"
           type: file
@@ -588,4 +586,4 @@ suricata:
     threshold-file: /etc/suricata/threshold.conf
     #include: include1.yaml
     #include: include2.yaml
-    
+    

salt/suricata/init.sls

@@ -16,7 +16,6 @@
 {% if sls in allowed_states %}
 
 {% set interface = salt['pillar.get']('sensor:interface', 'bond0') %}
-{% set ZEEKVER = salt['pillar.get']('global:mdengine', '') %}
 {% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %}
 {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %}
 {% set MANAGER = salt['grains.get']('master') %}
@@ -64,9 +63,10 @@ surilogdir:
 
 suridatadir:
   file.directory:
-    - name: /nsm/suricata
+    - name: /nsm/suricata/extracted
     - user: 940
     - group: 939
+    - makedirs: True
 
 surirulesync:
   file.recurse:
@@ -74,7 +74,6 @@ surirulesync:
     - source: salt://suricata/rules/
     - user: 940
     - group: 940
-    - show_changes: False
 
 surilogscript:
   file.managed:
@@ -148,6 +147,7 @@ so-suricata:
       - /opt/so/conf/suricata/rules:/etc/suricata/rules:ro
       - /opt/so/log/suricata/:/var/log/suricata/:rw
       - /nsm/suricata/:/nsm/:rw
+      - /nsm/suricata/extracted:/var/log/suricata//filestore:rw
       - /opt/so/conf/suricata/bpf:/etc/suricata/bpf:ro
     - network_mode: host
     - watch:
@@ -178,31 +178,10 @@ disable_so-suricata_so-status.conf:
     - month: '*'
     - dayweek: '*'
 
-so-suricata-eve-clean:
-  file.managed:
-    - name: /usr/sbin/so-suricata-eve-clean
-    - user: root
-    - group: root
-    - mode: 755
-    - template: jinja
-    - source: salt://suricata/cron/so-suricata-eve-clean
-
-# Add eve clean cron
-clean_suricata_eve_files:
-  cron.present:
-    - name: /usr/sbin/so-suricata-eve-clean > /dev/null 2>&1
-    - identifier: clean_suricata_eve_files
-    - user: root
-    - minute: '10'
-    - hour: '0'
-    - daymonth: '*'
-    - month: '*'
-    - dayweek: '*'
-
 {% else %}
 
 {{sls}}_state_not_allowed:
   test.fail_without_changes:
     - name: {{sls}}_state_not_allowed
 
-{% endif %}
+{% endif %}

salt/suricata/suricata_config.map.jinja

@@ -3,7 +3,9 @@
 {% from 'suricata/afpacket.map.jinja' import afpacket %}
 {% set suricata_pillar = salt['pillar.get']('suricata:config', {}) %}
 {% set default_evelog_index = [] %}
+{% set default_filestore_index = [] %}
 {% set surimeta_evelog_index = [] %}
+{% set surimeta_filestore_index = [] %}
 
 {% if salt['pillar.get']('sensor:hnsensor') %}
   {% load_yaml as homenet %}
@@ -33,19 +35,28 @@ HOME_NET: "[{{salt['pillar.get']('global:hnmanager', '')}}]"
   {% if 'eve-log' in li.keys() %}
     {% do default_evelog_index.append(loop.index0) %}
   {% endif %}
+  {% if 'file-store' in li.keys() %}
+    {% do default_filestore_index.append(loop.index0) %}
+  {% endif %}
 {% endfor %}
 {% set default_evelog_index = default_evelog_index[0]  %}
+{% set default_filestore_index = default_filestore_index[0]  %}
 
 {# Find the index of eve-log so it can be grabbed later #}
 {% for li in suricata_meta.suricata.config.outputs %}
   {% if 'eve-log' in li.keys() %}
     {% do surimeta_evelog_index.append(loop.index0) %}
   {% endif %}
+  {% if 'file-store' in li.keys() %}
+    {% do surimeta_filestore_index.append(loop.index0) %}
+  {% endif %}
 {% endfor %}
 {% set surimeta_evelog_index = surimeta_evelog_index[0] %}
+{% set surimeta_filestore_index = surimeta_filestore_index[0] %}
 
 {% if salt['pillar.get']('global:mdengine', 'ZEEK') == 'SURICATA' %}
   {% do suricata_defaults.suricata.config.outputs[default_evelog_index]['eve-log'].types.extend(suricata_meta.suricata.config.outputs[surimeta_evelog_index]['eve-log'].types) %}
+  {% do suricata_defaults.suricata.config.outputs[default_filestore_index]['file-store'].update({'enabled':suricata_meta.suricata.config.outputs[surimeta_filestore_index]['file-store']['enabled']}) %}
 {% endif %}
 
 {% do suricata_defaults.suricata.config.update(default_packet_size) %}

salt/suricata/suricata_meta.yaml

@@ -1,6 +1,20 @@
 suricata:
   config:
     outputs:
+      - file-store:
+          version: 2
+          enabled: "yes"
+          dir: /nsm/extracted
+          #write-fileinfo: "yes"
+          #force-filestore: "yes"
+          #stream-depth: 0
+          #max-open-files: 1000
+          #force-hash: [sha1, md5]
+          xff:
+              enabled: "no"
+              mode: extra-data
+              deployment: reverse
+              header: X-Forwarded-For
       - eve-log:
           types:
             - anomaly:
@@ -19,15 +33,15 @@ suricata:
                 enabled: "yes"
                 #requests: "no"
                 #responses: "no"
-                #formats: [detailed, grouped]
+                formats: [grouped]
                 #types: [a, aaaa, cname, mx, ns, ptr, txt]
             - tls:
                 extended: "yes"
                 #session-resumption: "no"
                 #custom: [subject, issuer, session_resumed, serial, fingerprint, sni, version, not_before, not_after, certificate, chain, ja3, ja3s]
             - files:
-                force-magic: "no"
+                force-magic: "yes"
-                #force-hash: [md5]
+                force-hash: [md5,sha1]
             #- drop:
             #    alerts: "yes"
             #    flows: all

setup/so-setup

@@ -336,52 +336,52 @@ minion_type=$(get_minion_type)
 set_default_log_size >> $setup_log 2>&1
 
 if [[ $is_helix ]]; then
-	RULESETUP=ETOPEN
+    RULESETUP=${RULESETUP:-ETOPEN}
-	NSMSETUP=BASIC
+	NSMSETUP=${NSMSETUP:-BASIC}
-	HNSENSOR=inherit
+	HNSENSOR=${HNSENSOR:-inherit}
-	MANAGERUPDATES=0
+	MANAGERUPDATES=${MANAGERUPDATES:-0}
 fi
 
 if [[ $is_helix || ( $is_manager && $is_node ) ]]; then
-	RULESETUP=ETOPEN
+    RULESETUP=${RULESETUP:-ETOPEN}
-	NSMSETUP=BASIC
+	NSMSETUP=${NSMSETUP:-BASIC}
 fi
 
 if [[ $is_manager && $is_node ]]; then
-	LSPIPELINEWORKERS=1
+	LSPIPELINEWORKERS=${LSPIPELINEWORKERS:-1}
-	LSPIPELINEBATCH=125
+	LSPIPELINEBATCH=${LSPIPELINEBATCH:-125}
-	LSINPUTTHREADS=1
+	LSINPUTTHREADS=${LSINPUTTHREADS:-1}
-	LSPIPELINEBATCH=125
+	LSPIPELINEWORKERS=${LSPIPELINEBATCH:-125}
-	NIDS=Suricata
+	NIDS=${NIDS:-Suricata}
-	ZEEKVERSION=ZEEK
+	ZEEKVERSION=${ZEEKVERSION:-ZEEK}
 fi
 
 if [[ $is_node ]]; then
-	CURCLOSEDAYS=30
+	CURCLOSEDAYS=${CURCLOSEDAYS:-30}
 fi
 
 if [[ $is_import ]]; then
-	PATCHSCHEDULENAME=auto
+	PATCHSCHEDULENAME=${PATCHSCHEDULENAME:-auto}
-	MTU=1500
+	MTU=${MTU:-1500}
-	RULESETUP=ETOPEN
+    RULESETUP=${RULESETUP:-ETOPEN}
-	NSMSETUP=BASIC
+	NSMSETUP=${NSMSETUP:-BASIC}
-	HNSENSOR=inherit
+	HNSENSOR=${HNSENSOR:-inherit}
-	MANAGERUPDATES=0
+	MANAGERUPDATES=${MANAGERUPDATES:-0}
-	MANAGERADV=BASIC
+	MANAGERADV=${MANAGERADV:-BASIC}
-	INTERFACE=bond0
+	INTERFACE=${INTERFACE:-bond0}
-	ZEEKVERSION=ZEEK
+	ZEEKVERSION=${ZEEKVERSION:-ZEEK}
-	NIDS=Suricata
+	NIDS=${NIDS:-Suricata}
-	RULESETUP=ETOPEN
+	RULESETUP=${RULESETUP:-ETOPEN}
-	GRAFANA=0
+	GRAFANA=${GRAFANA:-0}
-	OSQUERY=0
+	OSQUERY=${OSQUERY:-0}
-	WAZUH=0
+	WAZUH=${WAZUH:-0}
-	THEHIVE=0
+	THEHIVE=${THEHIVE:-0}
-	PLAYBOOK=0
+	PLAYBOOK=${PLAYBOOK:-0}
 fi
 
 if [[ $is_airgap ]]; then
-	PATCHSCHEDULENAME=manual
+	PATCHSCHEDULENAME=${PATCHSCHEDULENAME:-manual}
-	MANAGERUPDATES=0 
+	MANAGERUPDATES=${MANAGERUPDATES:-0} 
 fi
 
 # Start user prompts
@@ -708,31 +708,30 @@ set_redirect >> $setup_log 2>&1
 		
 		set_progress_str 60 "$(print_salt_state_apply 'manager')"
 		salt-call state.apply -l info manager >> $setup_log 2>&1
-	fi
 
-	set_progress_str 61 "$(print_salt_state_apply 'firewall')"
+		set_progress_str 61 "$(print_salt_state_apply 'idstools')"
-	salt-call state.apply -l info firewall >> $setup_log 2>&1
-
-	if [ $OS = 'centos' ]; then
-		set_progress_str 62 'Installing Yum utilities'
-		salt-call state.apply -l info yum.packages >> $setup_log 2>&1
-	fi
-
-	set_progress_str 62 "$(print_salt_state_apply 'common')"
-	salt-call state.apply -l info common >> $setup_log 2>&1
-
-	if [[ ! $is_helix ]]; then
-		set_progress_str 62 "$(print_salt_state_apply 'nginx')"
-		salt-call state.apply -l info nginx >> $setup_log 2>&1
-	fi
-
-	if [[ $is_manager || $is_helix || $is_import ]]; then
-		set_progress_str 63 "$(print_salt_state_apply 'idstools')"
 		create_local_nids_rules >> $setup_log 2>&1
 		salt-call state.apply -l info idstools >> $setup_log 2>&1
 
-		set_progress_str 63 "$(print_salt_state_apply 'suricata.manager')"
+		set_progress_str 61 "$(print_salt_state_apply 'suricata.manager')"
 		salt-call state.apply -l info suricata.manager >> $setup_log 2>&1
+
+	fi
+
+	set_progress_str 62 "$(print_salt_state_apply 'firewall')"
+	salt-call state.apply -l info firewall >> $setup_log 2>&1
+
+	if [ $OS = 'centos' ]; then
+		set_progress_str 63 'Installing Yum utilities'
+		salt-call state.apply -l info yum.packages >> $setup_log 2>&1
+	fi
+
+	set_progress_str 63 "$(print_salt_state_apply 'common')"
+	salt-call state.apply -l info common >> $setup_log 2>&1
+
+	if [[ ! $is_helix ]]; then
+		set_progress_str 64 "$(print_salt_state_apply 'nginx')"
+		salt-call state.apply -l info nginx >> $setup_log 2>&1
 	fi
 
 	if [[ $is_manager || $is_node || $is_import || $is_helix ]]; then
@@ -911,4 +910,4 @@ fi
 
 install_cleanup >> "$setup_log" 2>&1
 
-if [[ -z $SKIP_REBOOT ]]; then shutdown -r now; else exit; fi
+if [[ -z $SKIP_REBOOT ]]; then shutdown -r now; else exit; fi