Pull in Suricata changes

2025-12-06 17:22:49 +01:00 · 2021-02-19 11:01:15 -05:00
parent 4a510df205
commit b4b449aa14
19 changed files with 237 additions and 101 deletions
--- a/salt/elasticsearch/files/ingest/suricata.common
+++ b/salt/elasticsearch/files/ingest/suricata.common
@@ -15,6 +15,6 @@
    { "set":         { "field": "ingest.timestamp",        "value": "{{@timestamp}}"  } },
    { "date": { "field": "message2.timestamp", "target_field": "@timestamp", "formats": ["ISO8601", "UNIX"], "timezone": "UTC", "ignore_failure": true } },
    { "remove":{ "field": "agent", 						"ignore_failure": true	} },
-    { "pipeline": { "name": "suricata.{{dataset}}" } }
+    { "pipeline": { "if": "ctx?.dataset != null", "name": "suricata.{{dataset}}" } }
  ]
 }
--- a/salt/elasticsearch/files/ingest/suricata.dns
+++ b/salt/elasticsearch/files/ingest/suricata.dns
@@ -7,8 +7,13 @@
    { "rename": 	{ "field": "message2.dns.tx_id",		"target_field": "dns.id",	"ignore_missing": true 	} },
    { "rename": 	{ "field": "message2.dns.version", 		"target_field": "dns.version",		"ignore_missing": true 	} },
    { "rename": 	{ "field": "message2.dns.rrname", 		"target_field": "dns.query.name",		"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.grouped.A", 		"target_field": "dns.answers",			"ignore_missing": true 	} },
-
+    { "rename": 	{ "field": "message2.dns.flags", 		"target_field": "dns.flags",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.dns.qr", 		"target_field": "dns.qr",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.dns.rd", 		"target_field": "dns.recursion.desired",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.dns.ra", 		"target_field": "dns.recursion.available",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.dns.rcode", 		"target_field": "dns.response.code",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.grouped.A", 		"target_field": "dns.answers.data",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.grouped.CNAME", 		"target_field": "dns.answers.name",			"ignore_missing": true 	} },
    { "pipeline": { "name": "common" } }
  ]
 }
--- a/salt/elasticsearch/files/ingest/suricata.fileinfo
+++ b/salt/elasticsearch/files/ingest/suricata.fileinfo
@@ -1,8 +1,18 @@
 {
  "description" : "suricata.fileinfo",
  "processors" : [
+    { "set":      { "field": "dataset",        "value": "file"  } },
    { "rename": 	{ "field": "message2.proto", 		"target_field": "network.transport",		"ignore_missing": true 	} },
    { "rename": 	{ "field": "message2.app_proto", 		"target_field": "network.protocol",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.fileinfo.filename", 		"target_field": "file.name",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.fileinfo.gaps", 		"target_field": "file.bytes.missing",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.fileinfo.magic", 		"target_field": "file.mime_type",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.fileinfo.md5", 		"target_field": "hash.md5",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.fileinfo.sha1", 		"target_field": "hash.sha1",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.fileinfo.sid", 		"target_field": "rule.uuid",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.fileinfo.size", 		"target_field": "file.size",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.fileinfo.state", 		"target_field": "file.state",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.fileinfo.stored", 		"target_field": "file.saved",		"ignore_missing": true 	} },
    { "pipeline": { "name": "common" } }
  ]
 }
--- a/salt/elasticsearch/files/ingest/suricata.krb5
+++ b/salt/elasticsearch/files/ingest/suricata.krb5
@@ -1,8 +1,15 @@
 {
  "description" : "suricata.krb5",
  "processors" : [
+    { "set":      { "field": "dataset",        "value": "kerberos"  } },
    { "rename": 	{ "field": "message2.proto", 		"target_field": "network.transport",		"ignore_missing": true 	} },
    { "rename": 	{ "field": "message2.app_proto", 		"target_field": "network.protocol",		"ignore_missing": true 	} },
+    { "rename":         { "field": "message2.krb5.msg_type",                "target_field": "kerberos.request_type",             "ignore_missing": true  } },
+    { "rename":         { "field": "message2.krb5.cname",                "target_field": "kerberos.client",             "ignore_missing": true  } },
+    { "rename":         { "field": "message2.krb5.realm",                "target_field": "kerberos.realm",             "ignore_missing": true  } },
+    { "rename":         { "field": "message2.krb5.sname",                "target_field": "kerberos.service",             "ignore_missing": true  } },
+    { "rename":         { "field": "message2.krb5.encryption",                "target_field": "kerberos.ticket.cipher",             "ignore_missing": true  } },
+    { "rename":         { "field": "message2.krb.weak_encryption",                "target_field": "kerberos.weak_encryption",             "ignore_missing": true  } },
    { "pipeline": { "name": "common" } }
  ]
 }
--- a/salt/elasticsearch/files/ingest/suricata.rdp
+++ b/salt/elasticsearch/files/ingest/suricata.rdp
@@ -3,6 +3,26 @@
  "processors" : [
    { "rename": 	{ "field": "message2.proto", 		"target_field": "network.transport",		"ignore_missing": true 	} },
    { "rename": 	{ "field": "message2.app_proto", 		"target_field": "network.protocol",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.rdp.tx_id", 		"target_field": "rdp.tx_id",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.rdp.event_type", 		"target_field": "rdp.event_type",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.rdp.cookie", 		"target_field": "rdp.cookie",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.rdp.client.version", 		"target_field": "rdp.client_version",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.rdp.client.desktop_width", 		"target_field": "rdp.desktop__width",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.rdp.client.desktop__height", 		"target_field": "rdp.desktop__height",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.rdp.client.color_depth", 		"target_field": "rdp.requested_color_depth",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.rdp.client.keyboard_layout", 		"target_field": "rdp.keyboard_layout",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.rdp.client.build", 		"target_field": "rdp.client_build",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.rdp.client.client_name", 		"target_field": "client.name",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.rdp.client.keyboard_type", 		"target_field": "rdp.keyboard_type",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.rdp.client.function_keys", 		"target_field": "rdp.function_keys",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.rdp.client.product_id", 		"target_field": "rdp.product_id",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.rdp.client.capabilities", 		"target_field": "rdp.client_capabilities",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.rdp.client.id", 		"target_field": "rdp.client_id",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.rdp.channels", 		"target_field": "rdp.channels",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.rdp.server_supports", 		"target_field": "rdp.server_supports",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.rdp.x509_serials", 		"target_field": "rdp.x509_serials",		"ignore_missing": true 	} },
+
+
    { "pipeline": { "name": "common" } }
  ]
 }
--- a/salt/elasticsearch/files/ingest/suricata.smb
+++ b/salt/elasticsearch/files/ingest/suricata.smb
@@ -3,6 +3,30 @@
  "processors" : [
    { "rename": 	{ "field": "message2.proto", 		"target_field": "network.transport",		"ignore_missing": true 	} },
    { "rename": 	{ "field": "message2.app_proto", 		"target_field": "network.protocol",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.smb.id", 		"target_field": "smb.id",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.smb.dialect", 		"target_field": "smb.dialect",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.smb.command", 		"target_field": "smb.command",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.smb.status", 		"target_field": "smb.status",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.smb.status_code", 		"target_field": "smb.status_code",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.smb.session_id", 		"target_field": "smb.session_id",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.smb.tree_id", 		"target_field": "smb.tree_id",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.smb.filename", 		"target_field": "smb.filename",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.smb.disposition", 		"target_field": "smb.disposition",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.smb.access", 		"target_field": "smb.access",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.smb.created", 		"target_field": "smb.created",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.smb.accessed", 		"target_field": "smb.accessed",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.smb.modified", 		"target_field": "smb.modified ",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.smb.changed", 		"target_field": "smb.changed",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.smb.size", 		"target_field": "smb.size",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.smb.share", 		"target_field": "smb.share",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.smb.share_type", 		"target_field": "smb.share_type",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.client_dialects", 		"target_field": "smb.client_dialects",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.client_guid", 		"target_field": "smb.client_guid",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.server_guid", 		"target_field": "smb.server_guid",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.request.native_us", 		"target_field": "smb.request.native_us",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.request.native_lm", 		"target_field": "smb.request.native_lm",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.response.native_os", 		"target_field": "smb.response.native_os",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.response.native_lm", 		"target_field": "smb.response.native_lm",		"ignore_missing": true 	} },
    { "pipeline": { "name": "common" } }
  ]
 }
--- a/salt/elasticsearch/files/ingest/suricata.snmp
+++ b/salt/elasticsearch/files/ingest/suricata.snmp
@@ -5,6 +5,8 @@
    { "rename": 	{ "field": "message2.app_proto", 		"target_field": "network.protocol",		"ignore_missing": true 	} },
    { "rename": 	{ "field": "message2.snmp.version", 		"target_field": "snmp.version",		"ignore_missing": true 	} },
    { "rename": 	{ "field": "message2.snmp.community", 		"target_field": "snmp.community",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.snmp.pdu_type", 		"target_field": "snmp.pdu_type",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.snmp.vars", 		"target_field": "snmp.vars",		"ignore_missing": true 	} },
    { "pipeline": { "name": "common" } }
  ]
 }
--- a/salt/elasticsearch/files/ingest/suricata.tftp
+++ b/salt/elasticsearch/files/ingest/suricata.tftp
@@ -3,6 +3,10 @@
  "processors" : [
    { "rename": 	{ "field": "message2.proto", 		"target_field": "network.transport",		"ignore_missing": true 	} },
    { "rename": 	{ "field": "message2.app_proto", 		"target_field": "network.protocol",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.tftp.packet", 		"target_field": "tftp.packet",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.tftp.file", 		"target_field": "tftp.file",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.tftp.mode", 		"target_field": "tftp.mode",		"ignore_missing": true 	} },
+    
    { "pipeline": { "name": "common" } }
  ]
 }
--- a/salt/elasticsearch/files/ingest/suricata.tls
+++ b/salt/elasticsearch/files/ingest/suricata.tls
@@ -1,8 +1,22 @@
 {
  "description" : "suricata.tls",
  "processors" : [
+    { "set":      { "field": "dataset",        "value": "ssl"  } },
    { "rename": 	{ "field": "message2.proto", 		"target_field": "network.transport",		"ignore_missing": true 	} },
    { "rename": 	{ "field": "message2.app_proto", 		"target_field": "network.protocol",		"ignore_missing": true 	} },
+    { "rename":         { "field": "message2.tls.subject",                "target_field": "ssl.certificate.subject",             "ignore_missing": true  } },
+    { "rename":         { "field": "message2.tls.serial",                "target_field": "ssl.certificate.serial",             "ignore_missing": true  } },
+    { "rename":         { "field": "message2.tls.fingerprint",                "target_field": "ssl.certificate.fingerprint",             "ignore_missing": true  } },
+    { "rename":         { "field": "message2.tls.version",                "target_field": "ssl.certificate.version",             "ignore_missing": true  } },
+    { "rename":         { "field": "message2.tls.ja3.hash",                "target_field": "hash.ja3",             "ignore_missing": true  } },
+    { "rename":         { "field": "message2.tls.ja3.hash.string",                "target_field": "hash.ja3_string",             "ignore_missing": true  } },
+    { "rename":         { "field": "message2.tls.ja3s.hash",                "target_field": "hash.ja3s",             "ignore_missing": true  } },
+    { "rename":         { "field": "message2.tls.ja3s.hash.string",                "target_field": "hash.ja3s_string",             "ignore_missing": true  } },
+    { "rename":         { "field": "message2.tls.notbefore",                "target_field": "x509.certificate.not_valid_before",             "ignore_missing": true  } },
+    { "rename":         { "field": "message2.tls.notafter",                "target_field": "x509.certificate.not_valid_after",             "ignore_missing": true  } },
+    { "rename":         { "field": "message2.tls.sni",                "target_field": "ssl.server_name",             "ignore_missing": true  } },
+    { "rename":         { "field": "message2.tls.issuerdn",                "target_field": "ssl.certificate.issuer",             "ignore_missing": true  } },
+    { "rename":         { "field": "message2.tls.session_resumed",                "target_field": "ssl.session_resumed",             "ignore_missing": true  } },
    { "pipeline": { "name": "common" } }
  ]
 }
--- a/salt/idstools/etc/rulecat.conf
+++ b/salt/idstools/etc/rulecat.conf
@@ -3,17 +3,26 @@
 {%- set OINKCODE = salt['pillar.get']('idstools:config:oinkcode', '' ) -%}
 {%- set ISAIRGAP = salt['pillar.get']('global:airgap', 'False') -%}
 {%- set MANAGERIP = salt['pillar.get']('global:managerip', '') -%}
+{%- set ENGINE = salt['pillar.get']('global:mdengine', '') %}
 {%- if ISAIRGAP is sameas true -%}
 --merged=/opt/so/rules/nids/all.rules
 --local=/opt/so/rules/nids/local.rules
+{%- if ENGINE == "SURICATA" %}
+--local=/opt/so/rules/nids/sorules/extraction.rules
+--local=/opt/so/rules/nids/sorules/filters.rules
+{%- endif %}
 --url=http://{{ MANAGERIP }}:7788/rules/emerging-all.rules
 --disable=/opt/so/idstools/etc/disable.conf
 --enable=/opt/so/idstools/etc/enable.conf
 --modify=/opt/so/idstools/etc/modify.conf
 {%- else -%}
--suricata-version=5.0
+--suricata-version=6.0
 --merged=/opt/so/rules/nids/all.rules
 --local=/opt/so/rules/nids/local.rules
+{%- if ENGINE == "SURICATA" %}
+--local=/opt/so/rules/nids/sorules/extraction.rules
+--local=/opt/so/rules/nids/sorules/filters.rules
+{%- endif %}
 --disable=/opt/so/idstools/etc/disable.conf
 --enable=/opt/so/idstools/etc/enable.conf
 --modify=/opt/so/idstools/etc/modify.conf
--- a/salt/idstools/init.sls
+++ b/salt/idstools/init.sls
@@ -18,6 +18,7 @@
 {% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %}
 {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %}
 {% set MANAGER = salt['grains.get']('master') %}
+{% set ENGINE = salt['pillar.get']('global:mdengine', '') %}
 # IDSTools Setup
 idstoolsdir:
  file.directory:
@@ -55,14 +56,12 @@ rulesdir:
    - group: 939
    - makedirs: True

-# Don't show changes because all.rules can be large
 synclocalnidsrules:
  file.recurse:
    - name: /opt/so/rules/nids/
    - source: salt://idstools/
    - user: 939
    - group: 939
-    - show_changes: False
    - include_pat: 'E@.rules'

 so-idstools:
--- a/salt/idstools/sorules/extraction.rules
+++ b/salt/idstools/sorules/extraction.rules
@@ -0,0 +1,26 @@
+# Extract all PDF mime type
+alert http any any -> any any (msg:"FILE pdf detected"; filemagic:"PDF document"; filestore; sid:1100000; rev:1;)
+alert smtp any any -> any any (msg:"FILE pdf detected"; filemagic:"PDF document"; filestore; sid:1100001; rev:1;)
+alert nfs any any -> any any (msg:"FILE pdf detected"; filemagic:"PDF document"; filestore; sid:1100002; rev:1;)
+alert smb any any -> any any (msg:"FILE pdf detected"; filemagic:"PDF document"; filestore; sid:1100003; rev:1;)
+# Extract EXE/DLL file types
+alert http any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; sid:1100004; rev:1;)
+alert smtp any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; sid:1100005; rev:1;)
+alert nfs any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; sid:1100006; rev:1;)
+alert smb any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; sid:11000007; rev:1;)
+alert http any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; sid:1100008; rev:1;)
+alert smtp any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; sid:1100009; rev:1;)
+alert nfs any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; sid:1100010; rev:1;)
+alert smb any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; sid:11000011; rev:1;)
+
+# Extract all Zip files
+alert http any any -> any any (msg:"FILE ZIP detected"; filemagic:"Zip"; filestore; sid:1100012; rev:1;)
+alert smtp any any -> any any (msg:"FILE EXE detected"; filemagic:"Zip"; filestore; sid:1100013; rev:1;)
+alert nfs any any -> any any (msg:"FILE EXE detected"; filemagic:"Zip"; filestore; sid:1100014; rev:1;)
+alert smb any any -> any any (msg:"FILE EXE detected"; filemagic:"Zip"; filestore; sid:11000015; rev:1;)
+
+# Extract Word Docs
+alert http any any -> any any (msg:"FILE EXE detected"; filemagic:"Composite Document File V2 Document"; filestore; sid:1100016; rev:1;)
+alert smtp any any -> any any (msg:"FILE EXE detected"; filemagic:"Composite Document File V2 Document"; filestore; sid:1100017; rev:1;)
+alert nfs any any -> any any (msg:"FILE EXE detected"; filemagic:"Composite Document File V2 Document"; filestore; sid:1100018; rev:1;)
+alert smb any any -> any any (msg:"FILE EXE detected"; filemagic:"Composite Document File V2 Document"; filestore; sid:1100019; rev:1;)
--- a/salt/idstools/sorules/filer.rules
+++ b/salt/idstools/sorules/filer.rules
@@ -0,0 +1,3 @@
+# Start the filters at sid 1200000
+# Example of filtering out google.com from being dns logged.
+#config dns any any -> any any (dns.query; content:"google.com"; config: logging disable, type tx, scope tx; sid:1200000;)
--- a/salt/strelka/init.sls
+++ b/salt/strelka/init.sls
@@ -20,6 +20,7 @@
 {% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %}
 {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %}
 {% set STRELKA_RULES = salt['pillar.get']('strelka:rules', '1') %}
+{% set ENGINE = salt['pillar.get']('global:mdengine', '') %}

 # Strelka config
 strelkaconfdir:
@@ -188,6 +189,16 @@ strelka_zeek_extracted_sync_old:
    - name: '[ -d /nsm/zeek/extracted/complete/ ] && mv /nsm/zeek/extracted/complete/* /nsm/strelka/ > /dev/null 2>&1'
    - minute: '*'

+{% if ENGINE == "SURICATA" %}
+
+strelka_suricata_extracted_sync:
+  cron.present:
+    - user: root
+    - identifier: zeek-extracted-strelka-sync
+    - name: '[ -d /nsm/suricata/extracted/ ] && find /nsm/suricata/extracted/* -not \( -path /nsm/suricata/extracted/tmp -prune \) -type f -print0 | xargs -0 -I {} mv {} /nsm/strelka/unprocessed/ > /dev/null 2>&1'
+    - minute: '*'
+
+{% else %}
 strelka_zeek_extracted_sync:
  cron.present:
    - user: root
@@ -195,6 +206,7 @@ strelka_zeek_extracted_sync:
    - name: '[ -d /nsm/zeek/extracted/complete/ ] && mv /nsm/zeek/extracted/complete/* /nsm/strelka/unprocessed/ > /dev/null 2>&1'
    - minute: '*'

+{% endif %}
 {% else %}

 {{sls}}_state_not_allowed:
--- a/salt/suricata/defaults.yaml
+++ b/salt/suricata/defaults.yaml
@@ -156,8 +156,6 @@ suricata:
            mode: extra-data
            deployment: reverse
            header: X-Forwarded-For
-      - file-store:
-          enabled: "no"
      - tcp-data:
          enabled: "no"
          type: file
--- a/salt/suricata/init.sls
+++ b/salt/suricata/init.sls
@@ -16,7 +16,6 @@
 {% if sls in allowed_states %}

 {% set interface = salt['pillar.get']('sensor:interface', 'bond0') %}
-{% set ZEEKVER = salt['pillar.get']('global:mdengine', '') %}
 {% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %}
 {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %}
 {% set MANAGER = salt['grains.get']('master') %}
@@ -64,9 +63,10 @@ surilogdir:

 suridatadir:
  file.directory:
-    - name: /nsm/suricata
+    - name: /nsm/suricata/extracted
    - user: 940
    - group: 939
+    - makedirs: True

 surirulesync:
  file.recurse:
@@ -74,7 +74,6 @@ surirulesync:
    - source: salt://suricata/rules/
    - user: 940
    - group: 940
-    - show_changes: False

 surilogscript:
  file.managed:
@@ -148,6 +147,7 @@ so-suricata:
      - /opt/so/conf/suricata/rules:/etc/suricata/rules:ro
      - /opt/so/log/suricata/:/var/log/suricata/:rw
      - /nsm/suricata/:/nsm/:rw
+      - /nsm/suricata/extracted:/var/log/suricata//filestore:rw
      - /opt/so/conf/suricata/bpf:/etc/suricata/bpf:ro
    - network_mode: host
    - watch:
@@ -178,27 +178,6 @@ disable_so-suricata_so-status.conf:
    - month: '*'
    - dayweek: '*'

-so-suricata-eve-clean:
-  file.managed:
-    - name: /usr/sbin/so-suricata-eve-clean
-    - user: root
-    - group: root
-    - mode: 755
-    - template: jinja
-    - source: salt://suricata/cron/so-suricata-eve-clean
-
-# Add eve clean cron
-clean_suricata_eve_files:
-  cron.present:
-    - name: /usr/sbin/so-suricata-eve-clean > /dev/null 2>&1
-    - identifier: clean_suricata_eve_files
-    - user: root
-    - minute: '10'
-    - hour: '0'
-    - daymonth: '*'
-    - month: '*'
-    - dayweek: '*'
-
 {% else %}

 {{sls}}_state_not_allowed:
--- a/salt/suricata/suricata_config.map.jinja
+++ b/salt/suricata/suricata_config.map.jinja
@@ -3,7 +3,9 @@
 {% from 'suricata/afpacket.map.jinja' import afpacket %}
 {% set suricata_pillar = salt['pillar.get']('suricata:config', {}) %}
 {% set default_evelog_index = [] %}
+{% set default_filestore_index = [] %}
 {% set surimeta_evelog_index = [] %}
+{% set surimeta_filestore_index = [] %}

 {% if salt['pillar.get']('sensor:hnsensor') %}
  {% load_yaml as homenet %}
@@ -33,19 +35,28 @@ HOME_NET: "[{{salt['pillar.get']('global:hnmanager', '')}}]"
  {% if 'eve-log' in li.keys() %}
    {% do default_evelog_index.append(loop.index0) %}
  {% endif %}
+  {% if 'file-store' in li.keys() %}
+    {% do default_filestore_index.append(loop.index0) %}
+  {% endif %}
 {% endfor %}
 {% set default_evelog_index = default_evelog_index[0]  %}
+{% set default_filestore_index = default_filestore_index[0]  %}

 {# Find the index of eve-log so it can be grabbed later #}
 {% for li in suricata_meta.suricata.config.outputs %}
  {% if 'eve-log' in li.keys() %}
    {% do surimeta_evelog_index.append(loop.index0) %}
  {% endif %}
+  {% if 'file-store' in li.keys() %}
+    {% do surimeta_filestore_index.append(loop.index0) %}
+  {% endif %}
 {% endfor %}
 {% set surimeta_evelog_index = surimeta_evelog_index[0] %}
+{% set surimeta_filestore_index = surimeta_filestore_index[0] %}

 {% if salt['pillar.get']('global:mdengine', 'ZEEK') == 'SURICATA' %}
  {% do suricata_defaults.suricata.config.outputs[default_evelog_index]['eve-log'].types.extend(suricata_meta.suricata.config.outputs[surimeta_evelog_index]['eve-log'].types) %}
+  {% do suricata_defaults.suricata.config.outputs[default_filestore_index]['file-store'].update({'enabled':suricata_meta.suricata.config.outputs[surimeta_filestore_index]['file-store']['enabled']}) %}
 {% endif %}

 {% do suricata_defaults.suricata.config.update(default_packet_size) %}
--- a/salt/suricata/suricata_meta.yaml
+++ b/salt/suricata/suricata_meta.yaml
@@ -1,6 +1,20 @@
 suricata:
  config:
    outputs:
+      - file-store:
+          version: 2
+          enabled: "yes"
+          dir: /nsm/extracted
+          #write-fileinfo: "yes"
+          #force-filestore: "yes"
+          #stream-depth: 0
+          #max-open-files: 1000
+          #force-hash: [sha1, md5]
+          xff:
+              enabled: "no"
+              mode: extra-data
+              deployment: reverse
+              header: X-Forwarded-For
      - eve-log:
          types:
            - anomaly:
@@ -19,15 +33,15 @@ suricata:
                enabled: "yes"
                #requests: "no"
                #responses: "no"
-                #formats: [detailed, grouped]
+                formats: [grouped]
                #types: [a, aaaa, cname, mx, ns, ptr, txt]
            - tls:
                extended: "yes"
                #session-resumption: "no"
                #custom: [subject, issuer, session_resumed, serial, fingerprint, sni, version, not_before, not_after, certificate, chain, ja3, ja3s]
            - files:
-                force-magic: "no"
-                #force-hash: [md5]
+                force-magic: "yes"
+                force-hash: [md5,sha1]
            #- drop:
            #    alerts: "yes"
            #    flows: all
--- a/setup/so-setup
+++ b/setup/so-setup
@@ -336,52 +336,52 @@ minion_type=$(get_minion_type)
 set_default_log_size >> $setup_log 2>&1

 if [[ $is_helix ]]; then
-	RULESETUP=ETOPEN
-	NSMSETUP=BASIC
-	HNSENSOR=inherit
-	MANAGERUPDATES=0
+    RULESETUP=${RULESETUP:-ETOPEN}
+	NSMSETUP=${NSMSETUP:-BASIC}
+	HNSENSOR=${HNSENSOR:-inherit}
+	MANAGERUPDATES=${MANAGERUPDATES:-0}
 fi

 if [[ $is_helix || ( $is_manager && $is_node ) ]]; then
-	RULESETUP=ETOPEN
-	NSMSETUP=BASIC
+    RULESETUP=${RULESETUP:-ETOPEN}
+	NSMSETUP=${NSMSETUP:-BASIC}
 fi

 if [[ $is_manager && $is_node ]]; then
-	LSPIPELINEWORKERS=1
-	LSPIPELINEBATCH=125
-	LSINPUTTHREADS=1
-	LSPIPELINEBATCH=125
-	NIDS=Suricata
-	ZEEKVERSION=ZEEK
+	LSPIPELINEWORKERS=${LSPIPELINEWORKERS:-1}
+	LSPIPELINEBATCH=${LSPIPELINEBATCH:-125}
+	LSINPUTTHREADS=${LSINPUTTHREADS:-1}
+	LSPIPELINEWORKERS=${LSPIPELINEBATCH:-125}
+	NIDS=${NIDS:-Suricata}
+	ZEEKVERSION=${ZEEKVERSION:-ZEEK}
 fi

 if [[ $is_node ]]; then
-	CURCLOSEDAYS=30
+	CURCLOSEDAYS=${CURCLOSEDAYS:-30}
 fi

 if [[ $is_import ]]; then
-	PATCHSCHEDULENAME=auto
-	MTU=1500
-	RULESETUP=ETOPEN
-	NSMSETUP=BASIC
-	HNSENSOR=inherit
-	MANAGERUPDATES=0
-	MANAGERADV=BASIC
-	INTERFACE=bond0
-	ZEEKVERSION=ZEEK
-	NIDS=Suricata
-	RULESETUP=ETOPEN
-	GRAFANA=0
-	OSQUERY=0
-	WAZUH=0
-	THEHIVE=0
-	PLAYBOOK=0
+	PATCHSCHEDULENAME=${PATCHSCHEDULENAME:-auto}
+	MTU=${MTU:-1500}
+    RULESETUP=${RULESETUP:-ETOPEN}
+	NSMSETUP=${NSMSETUP:-BASIC}
+	HNSENSOR=${HNSENSOR:-inherit}
+	MANAGERUPDATES=${MANAGERUPDATES:-0}
+	MANAGERADV=${MANAGERADV:-BASIC}
+	INTERFACE=${INTERFACE:-bond0}
+	ZEEKVERSION=${ZEEKVERSION:-ZEEK}
+	NIDS=${NIDS:-Suricata}
+	RULESETUP=${RULESETUP:-ETOPEN}
+	GRAFANA=${GRAFANA:-0}
+	OSQUERY=${OSQUERY:-0}
+	WAZUH=${WAZUH:-0}
+	THEHIVE=${THEHIVE:-0}
+	PLAYBOOK=${PLAYBOOK:-0}
 fi

 if [[ $is_airgap ]]; then
-	PATCHSCHEDULENAME=manual
-	MANAGERUPDATES=0 
+	PATCHSCHEDULENAME=${PATCHSCHEDULENAME:-manual}
+	MANAGERUPDATES=${MANAGERUPDATES:-0} 
 fi

 # Start user prompts
@@ -708,31 +708,30 @@ set_redirect >> $setup_log 2>&1
 		
 		set_progress_str 60 "$(print_salt_state_apply 'manager')"
 		salt-call state.apply -l info manager >> $setup_log 2>&1
-	fi

-	set_progress_str 61 "$(print_salt_state_apply 'firewall')"
-	salt-call state.apply -l info firewall >> $setup_log 2>&1
-
-	if [ $OS = 'centos' ]; then
-		set_progress_str 62 'Installing Yum utilities'
-		salt-call state.apply -l info yum.packages >> $setup_log 2>&1
-	fi
-
-	set_progress_str 62 "$(print_salt_state_apply 'common')"
-	salt-call state.apply -l info common >> $setup_log 2>&1
-
-	if [[ ! $is_helix ]]; then
-		set_progress_str 62 "$(print_salt_state_apply 'nginx')"
-		salt-call state.apply -l info nginx >> $setup_log 2>&1
-	fi
-
-	if [[ $is_manager || $is_helix || $is_import ]]; then
-		set_progress_str 63 "$(print_salt_state_apply 'idstools')"
+		set_progress_str 61 "$(print_salt_state_apply 'idstools')"
 		create_local_nids_rules >> $setup_log 2>&1
 		salt-call state.apply -l info idstools >> $setup_log 2>&1

-		set_progress_str 63 "$(print_salt_state_apply 'suricata.manager')"
+		set_progress_str 61 "$(print_salt_state_apply 'suricata.manager')"
 		salt-call state.apply -l info suricata.manager >> $setup_log 2>&1
+
+	fi
+
+	set_progress_str 62 "$(print_salt_state_apply 'firewall')"
+	salt-call state.apply -l info firewall >> $setup_log 2>&1
+
+	if [ $OS = 'centos' ]; then
+		set_progress_str 63 'Installing Yum utilities'
+		salt-call state.apply -l info yum.packages >> $setup_log 2>&1
+	fi
+
+	set_progress_str 63 "$(print_salt_state_apply 'common')"
+	salt-call state.apply -l info common >> $setup_log 2>&1
+
+	if [[ ! $is_helix ]]; then
+		set_progress_str 64 "$(print_salt_state_apply 'nginx')"
+		salt-call state.apply -l info nginx >> $setup_log 2>&1
 	fi

 	if [[ $is_manager || $is_node || $is_import || $is_helix ]]; then