Merge branch 'dev' of https://github.com/Security-Onion-Solutions/securityonion-saltstack into dev

pillar/logstash/eval.sls

@@ -3,52 +3,19 @@ logstash:
     eval:
       config:
         - so/0800_input_eval.conf
-        - so/1000_preprocess_log_elapsed.conf
-        - so/1001_preprocess_syslogng.conf
         - so/1002_preprocess_json.conf
-        - so/1004_preprocess_syslog_types.conf
-        - so/1026_preprocess_dhcp.conf
-        - so/1029_preprocess_esxi.conf
-        - so/1030_preprocess_greensql.conf
-        - so/1031_preprocess_iis.conf
-        - so/1032_preprocess_mcafee.conf
         - so/1033_preprocess_snort.conf
-        - so/1034_preprocess_syslog.conf
-        - so/2000_network_flow.conf
-        - so/6002_syslog.conf
-        - so/6101_switch_brocade.conf
-        - so/6200_firewall_fortinet.conf
-        - so/6201_firewall_pfsense.conf
-        - so/6300_windows.conf
-        - so/6301_dns_windows.conf
-        - so/6400_suricata.conf
         - so/6500_ossec.conf
         - so/6501_ossec_sysmon.conf
         - so/6502_ossec_autoruns.conf
         - so/6600_winlogbeat_sysmon.conf
         - so/6700_winlogbeat.conf
         - so/7100_osquery_wel.conf
-        - so/7200_strelka.conf
-        - so/8001_postprocess_common_ip_augmentation.conf
-        - so/8007_postprocess_http.conf
-        - so/8200_postprocess_tagging.conf
-        - so/8998_postprocess_log_elapsed.conf
         - so/8999_postprocess_rename_type.conf
         - so/9000_output_bro.conf.jinja
-        - so/9001_output_switch.conf.jinja
         - so/9002_output_import.conf.jinja
-        - so/9004_output_flow.conf.jinja
-        - so/9026_output_dhcp.conf.jinja
-        - so/9029_output_esxi.conf.jinja
-        - so/9030_output_greensql.conf.jinja
-        - so/9031_output_iis.conf.jinja
-        - so/9032_output_mcafee.conf.jinja
         - so/9033_output_snort.conf.jinja
-        - so/9034_output_syslog.conf.jinja
         - so/9100_output_osquery.conf.jinja
-        - so/9200_output_firewall.conf.jinja
-        - so/9300_output_windows.conf.jinja
-        - so/9301_output_dns_windows.conf.jinja
         - so/9400_output_suricata.conf.jinja
         - so/9500_output_beats.conf.jinja
         - so/9600_output_ossec.conf.jinja

salt/elasticsearch/files/ingest/bro_conn

@@ -4,14 +4,14 @@
     { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
     { "rename": 	{ "field": "message2.uid", 	 	"target_field": "uid",			"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.orig_h", 		"path": "message2", 			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source_ip",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source.ip",		"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.orig_p", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source_port",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source.port",		"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.resp_h", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination_ip",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination.ip",	"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.resp_p", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination_port",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination.port",	"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.proto", 		"target_field": "protocol",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.proto", 		"target_field": "network.protocol",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.service", 		"target_field": "service",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.duration", 	"target_field": "duration",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.orig_bytes", 	"target_field": "original_bytes",	"ignore_missing": true 	} },

salt/elasticsearch/files/ingest/bro_dce_rpc

@@ -4,13 +4,13 @@
     { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
     { "rename": 	{ "field": "message2.uid", 	 	"target_field": "uid",			"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.orig_h", 		"path": "message2", 			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source_ip",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source.ip",		"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.orig_p", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source_port",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source.port",		"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.resp_h", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination_ip",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination.ip",	"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.resp_p", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination_port",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination.port",	"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.rtt", 		"target_field": "rtt",			"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.named_pipe",	"target_field": "named_pipe",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.endpoint", 	"target_field": "endpoint",		"ignore_missing": true 	} },

salt/elasticsearch/files/ingest/bro_dhcp

@@ -8,8 +8,8 @@
     { "rename": 	{ "field": "message2.lease_time", 	"target_field": "lease_time",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.trans_id", 	"target_field": "transaction_id",	"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.assigned_addr", 	"target_field": "assigned_ip",		"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.client_addr", 	"target_field": "source_ip",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.client_addr", 	"target_field": "source.ip",		"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.server_addr", 	"target_field": "destination_ip",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.server_addr", 	"target_field": "destination.ip",	"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.requested_addr", 	"target_field": "requested_ip",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.domain", 		"target_field": "domain_name",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.host_name",	"target_field": "hostname",		"ignore_missing": true 	} },

salt/elasticsearch/files/ingest/bro_dnp3

@@ -4,13 +4,13 @@
     { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
     { "rename": 	{ "field": "message2.uid", 	 	"target_field": "uid",			"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.orig_h", 		"path": "message2", 			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source_ip",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source.ip",		"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.orig_p", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source_port",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source.port",		"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.resp_h", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination_ip",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination.ip",	"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.resp_p", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination_port",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination.port",	"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.fc_request",	"target_field": "fc_request",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.fc_reply", 	"target_field": "fc_reply",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.iin", 		"target_field": "iin",			"ignore_missing": true 	} },

salt/elasticsearch/files/ingest/bro_dns

@@ -4,14 +4,14 @@
     { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
     { "rename": 	{ "field": "message2.uid", 	 	"target_field": "uid",			"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.orig_h", 		"path": "message2", 			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source_ip",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source.ip",		"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.orig_p", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source_port",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source.port",		"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.resp_h", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination_ip",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination.ip",	"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.resp_p", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination_port",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination.port",	"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.proto", 		"target_field": "protocol",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.proto", 		"target_field": "network.protocol",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.trans_id",		"target_field": "transaction_id",	"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.rtt",	 	"target_field": "rtt",			"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.query", 		"target_field": "query",		"ignore_missing": true 	} },

salt/elasticsearch/files/ingest/bro_dpd

@@ -4,14 +4,14 @@
     { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
     { "rename": 	{ "field": "message2.uid", 	 	"target_field": "uid",			"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.orig_h", 		"path": "message2", 			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source_ip",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source.ip",		"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.orig_p", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source_port",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source.port",		"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.resp_h", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination_ip",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination.ip",	"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.resp_p", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination_port",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination.port",	"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.proto", 		"target_field": "protocol",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.proto", 		"target_field": "network.protocol",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.analyzer", 	"target_field": "analyzer",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.failure_reason", 	"target_field": "failure_reason",	"ignore_missing": true 	} },
     { "pipeline":       { "name": "bro_common"                                                                                   } }

salt/elasticsearch/files/ingest/bro_files

@@ -4,11 +4,11 @@
     { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
     { "rename": 	{ "field": "message2.fuid", 	 	"target_field": "fuid",			"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.tx_hosts", 	"target_field": "file_ip",		"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.rx_hosts.0",	"target_field": "destination_ip",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.rx_hosts.0",	"target_field": "destination.ip",	"ignore_missing": true 	} },
     { "remove": 	{ "field": "message2.rx_hosts",							"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.conn_uids", 	"target_field": "uid",			"ignore_missing": true 	} },
     { "remove": 	{ "field": "source",								"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.source", 		"target_field": "source",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.source", 		"target_field": "file_source",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.depth",	 	"target_field": "depth",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.analyzers", 	"target_field": "analyzer",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.mime_type", 	"target_field": "mimetype",		"ignore_missing": true 	} },

salt/elasticsearch/files/ingest/bro_ftp

@@ -4,13 +4,13 @@
     { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
     { "rename": 	{ "field": "message2.uid", 	 	"target_field": "uid",			"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.orig_h", 		"path": "message2", 			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source_ip",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source.ip",		"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.orig_p", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source_port",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source.port",		"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.resp_h", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination_ip",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination.ip",	"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.resp_p", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination_port",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination.port",	"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.user",		"target_field": "username",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.password",		"target_field": "password",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.command", 		"target_field": "ftp_command",		"ignore_missing": true 	} },
@@ -22,11 +22,11 @@
     { "dot_expander": 	{ "field": "data_channel.passive", 		"path": "message2", 			"ignore_failure": true 	} },
     { "rename": 	{ "field": "message2.data_channel.passive","target_field": "data_channel_passive",	"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "data_channel.orig_h", 		"path": "message2", 			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.data_channel.orig_h","target_field": "data_channel_source_ip",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.data_channel.orig_h","target_field": "data_channel_source.ip",	"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "data_channel.resp_h", 		"path": "message2", 			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.data_channel.resp_h","target_field": "data_channel_destination_ip",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.data_channel.resp_h","target_field": "data_channel_destination.ip",	"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "data_channel.resp_p", 		"path": "message2", 			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.data_channel.resp_p","target_field": "data_channel_destination_port",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.data_channel.resp_p","target_field": "data_channel_destination.port",	"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.fuid",		"target_field": "fuid",		"ignore_missing": true 	} },
     { "pipeline":       { "name": "bro_common"                                                                                   } }
   ]

salt/elasticsearch/files/ingest/bro_http

@@ -4,13 +4,13 @@
     { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
     { "rename": 	{ "field": "message2.uid", 	 	"target_field": "uid",			"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.orig_h", 		"path": "message2", 			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source_ip",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source.ip",		"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.orig_p", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source_port",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source.port",		"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.resp_h", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination_ip",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination.ip",	"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.resp_p", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination_port",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination.port",	"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.trans_depth",	"target_field": "trans_depth",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.method", 		"target_field": "method",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.host", 		"target_field": "virtual_host",		"ignore_missing": true 	} },

salt/elasticsearch/files/ingest/bro_intel

@@ -4,13 +4,13 @@
     { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
     { "rename": 	{ "field": "message2.uid", 	 	"target_field": "uid",			"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.orig_h", 		"path": "message2", 			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source_ip",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source.ip",		"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.orig_p", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source_port",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source.port",		"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.resp_h", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination_ip",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination.ip",	"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.resp_p", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination_port",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination.port",	"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "seen.indicator", 		"path": "message2",			"ignore_failure": true 	} },
     { "rename": 	{ "field": "message2.seen.indicator", 	"target_field": "indicator",		"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "seen.indicator_type", 	"path": "message2",			"ignore_failure": true 	} },

salt/elasticsearch/files/ingest/bro_irc

@@ -4,13 +4,13 @@
     { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
     { "rename": 	{ "field": "message2.uid", 	 	"target_field": "uid",			"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.orig_h", 		"path": "message2", 			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source_ip",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source.ip",		"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.orig_p", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source_port",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source.port",		"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.resp_h", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination_ip",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination.ip",	"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.resp_p", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination_port",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination.port",	"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.nick", 		"target_field": "nick",			"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.user", 		"target_field": "irc_username",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.command", 		"target_field": "irc_command",		"ignore_missing": true 	} },

salt/elasticsearch/files/ingest/bro_kerberos

@@ -4,13 +4,13 @@
     { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
     { "rename": 	{ "field": "message2.uid", 	 	"target_field": "uid",			"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.orig_h", 		"path": "message2", 			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source_ip",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source.ip",		"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.orig_p", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source_port",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source.port",		"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.resp_h", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination_ip",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination.ip",	"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.resp_p", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination_port",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination.port",	"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.request_type", 	"target_field": "request_type",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.client", 		"target_field": "client",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.service", 		"target_field": "service",		"ignore_missing": true 	} },

salt/elasticsearch/files/ingest/bro_modbus

@@ -4,13 +4,13 @@
     { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
     { "rename": 	{ "field": "message2.uid", 	 	"target_field": "uid",			"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.orig_h", 		"path": "message2", 			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source_ip",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source.ip",		"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.orig_p", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source_port",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source.port",		"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.resp_h", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination_ip",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination.ip",	"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.resp_p", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination_port",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination.port",	"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.func", 		"target_field": "function",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.exception", 	"target_field": "exception",		"ignore_missing": true 	} },
     { "pipeline":       { "name": "bro_common"                                                                                   } }

salt/elasticsearch/files/ingest/bro_mysql

@@ -4,13 +4,13 @@
     { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
     { "rename": 	{ "field": "message2.uid", 	 	"target_field": "uid",			"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.orig_h", 		"path": "message2", 			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source_ip",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source.ip",		"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.orig_p", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source_port",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source.port",		"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.resp_h", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination_ip",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination.ip",	"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.resp_p", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination_port",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination.port",	"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.cmd", 		"target_field": "mysql_command",	"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.arg", 		"target_field": "mysql_argument",	"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.success", 		"target_field": "mysql_success",	"ignore_missing": true 	} },

salt/elasticsearch/files/ingest/bro_notice

@@ -6,17 +6,17 @@
     { "remove":		{ "field": "message2.src",		"ignore_failure": true						} },
     { "rename": 	{ "field": "message2.uid", 	 	"target_field": "uid",			"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.orig_h", 		"path": "message2", 			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source_ip",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source.ip",		"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.orig_p", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source_port",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source.port",		"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.resp_h", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination_ip",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination.ip",	"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.resp_p", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination_port",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination.port",	"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.fuid", 		"target_field": "fuid",			"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.mime", 		"target_field": "file_mime_type",	"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.desc", 		"target_field": "file_description",	"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.proto", 		"target_field": "protocol",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.proto", 		"target_field": "network.protocol",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.note", 		"target_field": "note",			"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.msg", 		"target_field": "msg",			"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.sub", 		"target_field": "sub_msg",		"ignore_missing": true 	} },

salt/elasticsearch/files/ingest/bro_ntlm

@@ -4,13 +4,13 @@
     { "json":		{ "field": "message",				"target_field": "message2",			"ignore_failure": true	} },
     { "rename": 	{ "field": "message2.uid", 	 		"target_field": "uid",				"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.orig_h", 			"path": "message2", 				"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.orig_h", 		"target_field": "source_ip",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_h", 		"target_field": "source.ip",			"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.orig_p", 			"path": "message2",				"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.orig_p", 		"target_field": "source_port",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_p", 		"target_field": "source.port",			"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.resp_h", 			"path": "message2",				"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.resp_h", 		"target_field": "destination_ip",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_h", 		"target_field": "destination.ip",		"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.resp_p", 			"path": "message2",				"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.resp_p", 		"target_field": "destination_port",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_p", 		"target_field": "destination.port",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.hostname",			"target_field": "hostname",			"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.domainname",		"target_field": "domain_name",			"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.success", 			"target_field": "ntlm_success",			"ignore_missing": true 	} },

salt/elasticsearch/files/ingest/bro_radius

@@ -4,13 +4,13 @@
     { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
     { "rename": 	{ "field": "message2.uid", 	 	"target_field": "uid",			"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.orig_h", 		"path": "message2", 			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source_ip",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source.ip",		"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.orig_p", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source_port",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source.port",		"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.resp_h", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination_ip",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination.ip",	"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.resp_p", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination_port",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination.port",	"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.username", 	"target_field": "username",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.mac", 		"target_field": "mac",			"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.framed_addr", 	"target_field": "framed_addr",		"ignore_missing": true 	} },

salt/elasticsearch/files/ingest/bro_rdp

@@ -4,13 +4,13 @@
     { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
     { "rename": 	{ "field": "message2.uid", 	 	"target_field": "uid",			"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.orig_h", 		"path": "message2", 			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source_ip",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source.ip",		"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.orig_p", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source_port",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source.port",		"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.resp_h", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination_ip",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination.ip",	"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.resp_p", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination_port",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination.port",	"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.cookie", 		"target_field": "cookie",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.result", 		"target_field": "result",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.security_protocol","target_field": "security_protocol",	"ignore_missing": true 	} },

salt/elasticsearch/files/ingest/bro_rfb

@@ -4,13 +4,13 @@
     { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
     { "rename": 	{ "field": "message2.uid", 	 	"target_field": "uid",			"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.orig_h", 		"path": "message2", 			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source_ip",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source.ip",		"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.orig_p", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source_port",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source.port",		"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.resp_h", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination_ip",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination.ip",	"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.resp_p", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination_port",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination.port",	"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.client_major_version",	"target_field": "client_major_version",	"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.client_minor_version",	"target_field": "client_minor_version",	"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.server_major_version",	"target_field": "server_major_version",	"ignore_missing": true 	} },

salt/elasticsearch/files/ingest/bro_signatures

@@ -4,13 +4,13 @@
     { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
     { "rename": 	{ "field": "message2.uid", 	 	"target_field": "uid",			"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.orig_h", 		"path": "message2", 			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source_ip",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source.ip",		"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.orig_p", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source_port",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source.port",		"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.resp_h", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination_ip",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination.ip",	"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.resp_p", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination_port",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination.port",	"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.note", 		"target_field": "note",			"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.sig_id", 		"target_field": "signature_id",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.event_msg", 	"target_field": "event_message",	"ignore_missing": true 	} },

salt/elasticsearch/files/ingest/bro_sip

@@ -4,13 +4,13 @@
     { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
     { "rename": 	{ "field": "message2.uid", 	 	"target_field": "uid",			"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.orig_h", 		"path": "message2", 			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source_ip",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source.ip",		"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.orig_p", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source_port",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source.port",		"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.resp_h", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination_ip",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination.ip",	"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.resp_p", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination_port",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination.port",	"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.trans_depth", 	"target_field": "trans_depth",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.method", 		"target_field": "method",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.uri", 		"target_field": "uri",			"ignore_missing": true 	} },

salt/elasticsearch/files/ingest/bro_smb_files

@@ -4,13 +4,13 @@
     { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
     { "rename": 	{ "field": "message2.uid", 	 	"target_field": "uid",			"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.orig_h", 		"path": "message2", 			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source_ip",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source.ip",		"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.orig_p", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source_port",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source.port",		"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.resp_h", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination_ip",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination.ip",	"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.resp_p", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination_port",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination.port",	"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.fuid", 		"target_field": "fuid",			"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.action", 		"target_field": "action",		"ignore_missing": true 	} },
     { "remove":		{ "field": "path",			"ignore_failure": true						} },

salt/elasticsearch/files/ingest/bro_smb_mapping

@@ -4,13 +4,13 @@
     { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
     { "rename": 	{ "field": "message2.uid", 	 	"target_field": "uid",			"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.orig_h", 		"path": "message2", 			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source_ip",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source.ip",		"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.orig_p", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source_port",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source.port",		"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.resp_h", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination_ip",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination.ip",	"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.resp_p", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination_port",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination.port",	"ignore_missing": true 	} },
     { "remove":		{ "field": "path",			"ignore_failure": true						} },
     { "rename": 	{ "field": "message2.path", 		"target_field": "path",			"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.service", 		"target_field": "service",		"ignore_missing": true 	} },

salt/elasticsearch/files/ingest/bro_smtp

@@ -5,13 +5,13 @@
     { "remove":		{ "field": "path",			"ignore_failure": true						} },
     { "rename": 	{ "field": "message2.uid", 	 	"target_field": "uid",			"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.orig_h", 		"path": "message2", 			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source_ip",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source.ip",		"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.orig_p", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source_port",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source.port",		"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.resp_h", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination_ip",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination.ip",	"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.resp_p", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination_port",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination.port",	"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.trans_depth", 	"target_field": "trans_depth",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.helo", 		"target_field": "helo",			"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.mailfrom", 	"target_field": "mail_from",		"ignore_missing": true 	} },

salt/elasticsearch/files/ingest/bro_snmp

@@ -4,13 +4,13 @@
     { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
     { "rename": 	{ "field": "message2.uid", 	 	"target_field": "uid",			"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.orig_h", 		"path": "message2", 			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source_ip",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source.ip",		"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.orig_p", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source_port",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source.port",		"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.resp_h", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination_ip",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination.ip",	"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.resp_p", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination_port",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination.port",	"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.duration",		"target_field": "duration",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.version", 		"target_field": "version",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.community", 	"target_field": "community",		"ignore_missing": true 	} },

salt/elasticsearch/files/ingest/bro_socks

@@ -4,13 +4,13 @@
     { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
     { "rename": 	{ "field": "message2.uid", 	 	"target_field": "uid",			"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.orig_h", 		"path": "message2", 			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source_ip",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source.ip",		"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.orig_p", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source_port",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source.port",		"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.resp_h", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination_ip",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination.ip",	"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.resp_p", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination_port",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination.port",	"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.version", 		"target_field": "version",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.user", 		"target_field": "user",			"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.password", 	"target_field": "password",		"ignore_missing": true 	} },

salt/elasticsearch/files/ingest/bro_software

@@ -13,8 +13,8 @@
     { "rename": 	{ "field": "message2.version.minor3", 	"target_field": "version_minor3",	"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "version.addl", 		"path": "message2", 			"ignore_failure": true 	} },
     { "rename": 	{ "field": "message2.version.addl", 	"target_field": "version_additional_info",	"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.host", 		"target_field": "source_ip",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.host", 		"target_field": "source.ip",		"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.host_p", 		"target_field": "source_port",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.host_p", 		"target_field": "source.port",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.software_type", 	"target_field": "software_type",	"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.name", 		"target_field": "name",			"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.unparsed_version",	"target_field": "unparsed_version",	"ignore_missing": true 	} },

salt/elasticsearch/files/ingest/bro_ssh

@@ -3,13 +3,13 @@
   "processors" : [
     { "json":		{ "field": "message",				"target_field": "message2",			"ignore_failure": true	} },
     { "dot_expander": 	{ "field": "id.orig_h", 			"path": "message2", 				"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.orig_h", 		"target_field": "source_ip",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_h", 		"target_field": "source.ip",			"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.orig_p", 			"path": "message2",				"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.orig_p", 		"target_field": "source_port",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_p", 		"target_field": "source.port",			"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.resp_h", 			"path": "message2",				"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.resp_h", 		"target_field": "destination_ip",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_h", 		"target_field": "destination.ip",		"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.resp_p", 			"path": "message2",				"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.resp_p", 		"target_field": "destination_port",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_p", 		"target_field": "destination.port",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.version", 			"target_field": "version",			"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.uid", 			"target_field": "uid",				"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.hassh", 			"target_field": "hassh",			"ignore_missing": true 	} },

salt/elasticsearch/files/ingest/bro_ssl

@@ -4,13 +4,13 @@
     { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
     { "rename": 	{ "field": "message2.uid", 	 	"target_field": "uid",			"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.orig_h", 		"path": "message2", 			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source_ip",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source.ip",		"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.orig_p", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source_port",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source.port",		"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.resp_h", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination_ip",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination.ip",	"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.resp_p", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination_port",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination.port",	"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.version", 		"target_field": "version",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.cipher", 		"target_field": "cipher",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.curve", 		"target_field": "curve",		"ignore_missing": true 	} },

salt/elasticsearch/files/ingest/bro_syslog

@@ -4,14 +4,14 @@
     { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
     { "rename": 	{ "field": "message2.uid", 	 	"target_field": "uid",			"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.orig_h", 		"path": "message2", 			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source_ip",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source.ip",		"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.orig_p", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source_port",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source.port",		"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.resp_h", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination_ip",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination.ip",	"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.resp_p", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination_port",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination.port",	"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.proto", 		"target_field": "protocol",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.proto", 		"target_field": "network.protocol",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.facility", 	"target_field": "facility",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.severity", 	"target_field": "severity",		"ignore_missing": true 	} },
     { "remove":		{ "field": "message",			"ignore_failure": true						} },

salt/elasticsearch/files/ingest/bro_tunnels

@@ -4,13 +4,13 @@
     { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
     { "rename": 	{ "field": "message2.uid", 	 	"target_field": "uid",			"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.orig_h", 		"path": "message2", 			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source_ip",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source.ip",		"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.orig_p", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source_port",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source.port",		"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.resp_h", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination_ip",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination.ip",	"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.resp_p", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination_port",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination.port",	"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.tunnel_type", 	"target_field": "tunnel_type",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.action", 		"target_field": "action",		"ignore_missing": true 	} },
     { "pipeline":       { "name": "bro_common"                                                                                   } }

salt/elasticsearch/files/ingest/bro_weird

@@ -4,13 +4,13 @@
     { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
     { "rename": 	{ "field": "message2.uid", 	 	"target_field": "uid",			"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.orig_h", 		"path": "message2", 			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source_ip",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source.ip",		"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.orig_p", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source_port",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source.port",		"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.resp_h", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination_ip",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination.ip",	"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "id.resp_p", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination_port",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination.port",	"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.name", 		"target_field": "name",			"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.addl", 		"target_field": "additional_info",	"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.notice", 		"target_field": "notice",		"ignore_missing": true 	} },

salt/elasticsearch/files/ingest/common

@@ -10,7 +10,7 @@
     },
     { 	
 	"geoip":  { 
-		"field": "destination_ip",
+		"field": "destination.ip",
 		"target_field": "destination_geo",
 		"database_file": "GeoLite2-City.mmdb",
 		"ignore_missing": true, 
@@ -19,7 +19,7 @@
     },
     { 
 	"geoip":  {
-		"field": "source_ip",
+		"field": "source.ip",
 		"target_field": "source_geo",
 		"database_file": "GeoLite2-City.mmdb",
 		"ignore_missing": true,

salt/elasticsearch/files/ingest/strelka

@@ -0,0 +1,12 @@
+{
+  "description" : "strelka",
+  "processors" : [
+    { "json":           { "field": "message",                        "target_field": "message2",                   "ignore_failure": true  } },
+    { "rename":         { "field": "message2.file",        "target_field": "file",          "ignore_missing": true  } },
+    { "rename":         { "field": "message2.scan",        "target_field": "scan",          "ignore_missing": true  } },
+    { "rename":         { "field": "message2.request",        "target_field": "request",          "ignore_missing": true  } },
+    { "rename":         { "field": "scan.hash",        "target_field": "file.hash",          "ignore_missing": true  } },
+    { "remove":         { "field": ["host", "path"],                                         "ignore_missing": true  } },
+    { "pipeline":       { "name": "common"                                                                                   } }
+  ]
+}

salt/logstash/init.sls

@@ -85,12 +85,6 @@ lspipelinedir:
     - group: 939
 
 {% for PL in PIPELINES %}
-ls_pipeline_{{PL}}:
-  file.directory:
-    - name: /opt/so/conf/logstash/pipelines/{{PL}}
-    - user: 931
-    - group: 939
-
   {% for CONFIGFILE in PIPELINES[PL].config %}
 ls_pipeline_{{PL}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }}:
   file.managed:
@@ -103,10 +97,23 @@ ls_pipeline_{{PL}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }}:
     {% endif %}
     - user: 931
     - group: 939
+    - makedirs: True
   {% endfor %}
+
+ls_pipeline_{{PL}}:
+  file.directory:
+    - name: /opt/so/conf/logstash/pipelines/{{PL}}
+    - user: 931
+    - group: 939
+    - require:
+  {% for CONFIGFILE in PIPELINES[PL].config %}
+      - file: ls_pipeline_{{PL}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }}
+  {% endfor %}
+    - clean: True
+
 {% endfor %}
 
-#sync templates to /opt/so/conf/logstash/etc/ here
+#sync templates to /opt/so/conf/logstash/etc
 {% for TEMPLATE in TEMPLATES %}
 ls_template_{{TEMPLATE.split('.')[0] | replace("/","_") }}:
   file.managed:
@@ -129,7 +136,7 @@ lspipelinesyml:
     - defaults:
         pipelines: {{ PIPELINES }}
 
-# Copy down all the configs including custom - TODO add watch restart
+# Copy down all the configs
 lsetcsync:
   file.recurse:
     - name: /opt/so/conf/logstash/etc
@@ -137,6 +144,11 @@ lsetcsync:
     - user: 931
     - group: 939
     - template: jinja
+    - clean: True
+    - require:
+{% for TEMPLATE in TEMPLATES %}
+      - file: ls_template_{{TEMPLATE.split('.')[0] | replace("/","_") }}
+{% endfor %}
     - exclude_pat: pipelines*
 
 # Create the import directory
@@ -176,12 +188,15 @@ so-logstash:
       - {{ BINDING }}
 {% endfor %}
     - binds:
+{% for TEMPLATE in TEMPLATES %}
+  {% if 'jinja' in TEMPLATE.split('.')[-1] %}
+      - /opt/so/conf/logstash/etc/{{TEMPLATE.split('/')[1] | replace(".jinja", "")}}:/{{TEMPLATE.split('/')[1] | replace(".jinja", "")}}:ro
+  {% else %}
+      - /opt/so/conf/logstash/etc/{{TEMPLATE.split('/')[1]}}:/{{TEMPLATE.split('/')[1]}}:ro
+  {% endif %}
+{% endfor %}
       - /opt/so/conf/logstash/etc/log4j2.properties:/usr/share/logstash/config/log4j2.properties:ro
       - /opt/so/conf/logstash/etc/logstash.yml:/usr/share/logstash/config/logstash.yml:ro
-      - /opt/so/conf/logstash/etc/logstash-template.json:/logstash-template.json:ro
-      - /opt/so/conf/logstash/etc/logstash-ossec-template.json:/logstash-ossec-template.json:ro
-      - /opt/so/conf/logstash/etc/logstash-strelka-template.json:/logstash-strelka-template.json:ro
-      - /opt/so/conf/logstash/etc/beats-template.json:/beats-template.json:ro
       - /opt/so/conf/logstash/etc/pipelines.yml:/usr/share/logstash/config/pipelines.yml
       - /opt/so/conf/logstash/pipelines:/usr/share/logstash/pipelines:ro
       - /opt/so/rules:/etc/nsm/rules:ro
@@ -201,6 +216,14 @@ so-logstash:
       - /opt/so/log/strelka:/strelka:ro
       {%- endif %}
     - watch:
-      - file: /opt/so/conf/logstash/etc
+      - file: lsetcsync
-      - file: /opt/so/conf/logstash/pipelines
+{% for PL in PIPELINES %}
-      #- file: /opt/so/conf/logstash/rulesets
+      - file: ls_pipeline_{{PL}}
+  {% for CONFIGFILE in PIPELINES[PL].config %}
+      - file: ls_pipeline_{{PL}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }}
+  {% endfor %}
+{% endfor %}
+{% for TEMPLATE in TEMPLATES %}
+      - file: ls_template_{{TEMPLATE.split('.')[0] | replace("/","_") }}
+{% endfor %}
+#     - file: /opt/so/conf/logstash/rulesets

salt/logstash/pipelines/config/so/1033_preprocess_snort.conf

@@ -1,181 +1,125 @@
-# Author: Justin Henderson
-#         SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
-# Updated by: Doug Burks
-# Last Update: 3/15/2018
-
 filter {
-  if [type] == "ids" {
+  if [engine] == "suricata" {
-    # This is the initial parsing of the log
+      json {
-    if [engine] == "suricata" {
+        source => "message"
-        json {
+      }
-            source => "message"
+      mutate {
+      # Make this compatible with event.id as a string
+      convert => { "[flow_id]" => "string" }
+      rename => {
+        "proto" => "[network][transport]"
+        "event_type" => "[event][dataset]"
+        "flow_id" => "[event][id]"
+        "community_id" => "[network][community_id]"
+      }
+      lowercase => [ "[network][transport]" ]
+      merge => {"[event][id]" => "[related][id]" }
+      add_field => {
+        "[related][domain]" => []
+        "[related][ip]" => []
+        "[related][id]" => []
+        "[event][module]" => "suricata"
+        "[event][created]" => "%{[@timestamp]}"
+        "[event][version]" => "1.0.0"
+        "[event][category]" => "network"
+      }
+    }
+
+    # Set the timestamp from the event
+    date {
+      match => [ "timestamp", "ISO8601" ]
+      tag_on_failure => [ "_dateparsefailure", "_parsefailure", "_suricata_dateparsefailure" ]
+      remove_field => [ "timestamp" ]
+    }
+
+    # Suricata uses top-level src/dest to track flow
+    if [src_ip] {
+      mutate {
+        rename => {
+          "[src_ip]" => "[source][ip]"
+          "[src_port]" => "[source][port]"
         }
+        merge => { "[related][ip]" => "[source][ip]" }
+      }
+    }
+    if [dest_ip] {
+      mutate {
+        rename => {
+          "[dest_ip]" => "[destination][ip]"
+          "[dest_port]" => "[destination][port]"
+        }
+        merge => { "[related][ip]" => "[destination][ip]" }
+      }
+    }
+
+    if [vlan] {
+      mutate {
+        rename => { "[vlan]" => "[vlan][id]" }
+      }
+    }
+    if [app_proto] {
+      if [app_proto] == "failed" {
+        # delete failed detections to be consistent with zeek
+        mutate { rename => { "app_proto" => "[error][message]" } }
+      }
+      else {
+        mutate { rename => {"app_proto" => "[network][protocol]"}}
+      }
+    }
+    if [event_type] == "alert" {
+      if [alert][severity] == 1 {
         mutate {
-	    rename => { "alert" => "orig_alert" } 
+          add_field => { "severity" => "High" }
-            rename => { "[orig_alert][gid]" => "gid" }
+        }
-            rename => { "[orig_alert][signature_id]" => "sid" }
+      }
-            rename => { "[orig_alert][rev]" => "rev" }
+      if [alert][severity] == 2 {
-            rename => { "[orig_alert][signature]" => "alert" }
+        mutate {
-            rename => { "[orig_alert][category]" => "classification" }
+          add_field => { "severity" => "Medium" }
-            rename => { "[orig_alert][severity]" => "priority" }
+        }
-	    rename => { "[orig_alert][rule]" => "rule_signature" }
+      }
-            rename => { "app_proto" => "application_protocol" }
+      if [alert][severity] == 3 {
-            rename => { "dest_ip" => "destination_ip" }
+        mutate {
-            rename => { "dest_port" => "destination_port" }
+          add_field => { "severity" => "Low" }
-            rename => { "in_iface" => "interface" }
+        }
-            rename => { "proto" => "protocol" }
+      }
-            rename => { "src_ip" => "source_ip" }
+          # If the alert is a Snort GPL alert break it apart for easier reading and categorization
-            rename => { "src_port" => "source_port" }
+      if [alert][signature] =~ "GPL " {
-	    #rename => { "[fileinfo][filename]" => "filename" }
+            # This will parse out the category type from the alert
-            #rename => { "[fileinfo][gaps]" => "gaps" }
-            #rename => { "[fileinfo][size]" => "size" }
-            #rename => { "[fileinfo][state]" => "state" }
-            #rename => { "[fileinfo][stored]" => "stored" }
-            #rename => { "[fileinfo][tx_id]" => "tx_id" }
-            #rename => { "[flow][age]" => "duration" }
-            #rename => { "[flow][alerted]" => "flow_alerted" }
-            #rename => { "[flow][bytes_toclient]" => "bytes_to_client" }
-            #rename => { "[flow][bytes_toserver]" => "bytes_to_server" }
-            #rename => { "[flow][end]" => "flow_end" }
-            #rename => { "[flow][pkts_toclient]" => "packets_to_client" }
-            #rename => { "[flow][pkts_toserver]" => "packets_to_server" }
-            #rename => { "[flow][reason]" => "reason" }
-            #rename => { "[flow][start]" => "flow_start" }
-            #rename => { "[flow][state]" => "state" }
-            #rename => { "[netflow][age]" => "duration" }
-            #rename => { "[netflow][bytes]" => "bytes" }
-            #rename => { "[netflow][end]" => "netflow_end" }
-            #rename => { "[netflow][start]" => "netflow_start" }
-            #rename => { "[netflow][pkts]" => "packets" }
-            rename => { "[alert][action]" => "action" }
-            rename => { "[alert][category]" => "category" }
-            rename => { "[alert][gid]" => "gid" }
-            rename => { "[alert][rev]" => "rev" }
-            rename => { "[alert][severity]" => "severity" }
-            rename => { "[alert][signature]" => "signature" }
-            rename => { "[alert][signature_id]" => "sid" }
-	    #rename => { "[dns][aa]" => "aa" }
-            #rename => { "[dns][flags]" => "flags" }
-	    #rename => { "[dns][id]" => "id" }
-	    #rename => { "[dns][qr]" => "qr" }
-            #rename => { "[dns][rcode]" => "rcode_name" }
-	    #rename => { "[dns][rrname]" => "rrname" }
-	    #rename => { "[dns][rrtype]" => "rrtype" }
-            #rename => { "[dns][tx_id]" => "tx_id" }
-	    #rename => { "[dns][type]" => "record_type" }
-	    #rename => { "[dns][version]" => "version" }
-            rename => { "[http][hostname]" => "virtual_host" }
-            rename => { "[http][http_content_type]" => "content_type" }
-            rename => { "[http][http_port]" => "http_port" }
-            rename => { "[http][http_method]" => "method" }
-	    rename => { "[http][http_user_agent]" => "useragent" }
-            #rename => { "[http][length]" => "payload_length" }
-            #rename => { "[http][protocol]" => "http_version" }
-            rename => { "[http][status]" => "status_message" }
-            rename => { "[http][url]" => "url" }
-            #rename => { "[metadata][flowbits]" => "flowbits" }
-            rename => { "[tls][fingerprint]" => "certificate_serial_number" }
-            rename => { "[tls][issuerdn]" => "issuer_distinguished_name" }
-            rename => { "[tls][notafter]" => "certificate_not_valid_after" }
-            rename => { "[tls][notbefore]" => "certificate_not_valid_before" }
-            rename => { "[tls][subject]" => "certificate_common_name" }
-            rename => { "[tls][version]" => "tls_version" }
-	    rename => { "event_type" => "ids_event_type" }
-	    remove_field => [ "offset", "orig_alert", "beat", "input", "prospector" ]
-	    remove_tag => [ "beats_input_codec_plain_applied" ]
-	    add_tag => [ "eve" ]
-             
-	}
-    } else {
         grok {
-          match => ["message", "\[%{INT:gid}:%{INT:sid}:%{INT:rev}\]\s%{DATA:alert}\[Classification:\s+%{DATA:classification}\]\s+\[Priority:\s+%{INT:priority}\]:\s+<%{DATA:interface}>\s+{%{DATA:protocol}}\s+(?:%{IPV4:source_ip}|%{IPV6:source_ip}):%{INT:source_port}\s+->\s+(?:%{IPV4:destination_ip}|%{IPV6:destination_ip}):%{INT:destination_port}",
+          match => { "[alert][signature]" => "GPL\s+%{DATA:category}\s" }
-		"message", "\[%{INT:gid}:%{INT:sid}:%{INT:rev}\]\s%{DATA:alert}\[Classification:\s+%{DATA:classification}\]\s+\[Priority:\s+%{INT:priority}\]:\s+<%{DATA:interface}>\s+{%{DATA:protocol}}\s(?:%{IPV4:source_ip}|%{IPV6:source_ip})\s+->\s+(?:%{IPV4:destination_ip}|%{IPV6:destination_ip})", 
-		"message", "\[%{INT:gid}:%{INT:sid}:%{INT:rev}\]\s%{DATA:alert}\[Classification:\s+%{DATA:classification}\]\s+\[Priority:\s+%{INT:priority}\]:\s+{%{DATA:protocol}}\s+(?:%{IPV4:source_ip}|%{IPV6:source_ip}):%{INT:source_port}\s+->\s+%{IPV4:destination_ip}:%{INT:destination_port}", 
-		"message", "\[%{INT:gid}:%{INT:sid}:%{INT:rev}\]\s%{DATA:alert}\[Classification:\s+%{DATA:classification}\]\s+\[Priority:\s+%{INT:priority}\]:\s+{%{DATA:protocol}}\s(?:%{IPV4:source_ip}|%{IPV6:source_ip})\s+->\s+(?:%{IPV4:destination_ip}|%{IPV6:destination_ip})",
-		"message", "\[%{INT:gid}:%{INT:sid}:%{INT:rev}\]\s%{DATA:alert}\[Classification:\s+%{DATA:classification}\]\s+\[Priority:\s+%{INT:priority}\]:\s+{%{DATA:protocol}}\s+(?:%{IPV4:source_ip}|%{IPV6:source_ip}):%{INT:source_port}\s+->\s+(?:%{IPV4:destination_ip}|%{IPV6:destination_ip}):%{INT:destination_port}",
-               "message", "\[%{INT:gid}:%{INT:sid}:%{INT:rev}\]\s%{DATA:alert}\[Classification:\s+%{DATA:classification}\]\s+\[Priority:\s+%{INT:priority}\]:\s+{%{DATA:protocol}}\s(?:%{IPV4:source_ip}|%{IPV6:source_ip})\s+->\s+(?:%{IPV4:source_ip}|%{IPV6:source_ip})",
-		"message", "\[%{INT:gid}:%{INT:sid}:%{INT:rev}\]\s%{DATA:alert}\[Classification:\s+%{DATA:classification}\]\s+\[Priority:\s+%{INT:priority}\]:\s+{%{DATA:protocol}}",
-		"message", "\A%{TIME} pid\(%{INT}\)  Alert Received: %{INT} %{INT:priority} %{DATA:classification} %{DATA:interface} \{%{DATA:timestamp}} %{INT} %{INT} \{%{DATA:alert}} %{IP:source_ip} %{IP:destination_ip} %{INT:protocol} %{INT:source_port} %{INT:destination_port} %{INT:gid} %{INT:sid} %{INT:rev} %{INT} %{INT}\Z",
-                "message", "%{GREEDYDATA:alert}"]
         }
-    }
+                # This will store the category
-    if [timestamp] {
         mutate {
-                add_field => { "logstash_timestamp" => "%{@timestamp}" }
+          add_field => { "rule_type" => "Snort GPL" }
+          lowercase => [ "category" ]
         }
-        mutate {
-                convert => { "logstash_timestamp" => "string" }
-        }
-        date {
-                match => [ "timestamp", "ISO8601" ]
-        }
-        mutate {
-                rename => { "logstash_timestamp" => "timestamp" }
-        }
-    }
-	
-	# If the alert is a Snort GPL alert break it apart for easier reading and categorization
-    if [alert] =~ "GPL " {
-	  # This will parse out the category type from the alert
-      grok {
-        match => { "alert" => "GPL\s+%{DATA:category}\s" }
       }
-	  # This will store the category
+          # If the alert is an Emerging Threat alert break it apart for easier reading and categorization
-      mutate {
+      if [alert][signature] =~ "ET " {
-        add_field => { "rule_type" => "Snort GPL" }
+            # This will parse out the category type from the alert
-        lowercase => [ "category"]
+        grok {
+          match => { "[alert][signature]" => "ET\s+%{DATA:category}\s" }
+        }
+                # This will store the category
+        mutate {
+          add_field => { "rule_type" => "Emerging Threats" }
+          lowercase => [ "category" ]
         }
-    }
-	# If the alert is an Emerging Threat alert break it apart for easier reading and categorization
-    if [alert] =~ "ET " {
-	  # This will parse out the category type from the alert
-      grok {
-        match => { "alert" => "ET\s+%{DATA:category}\s" }
       }
-	  # This will store the category
+          # This section adds URLs to lookup information about a rule online
-      mutate {
+      if [rule_type] == "Snort GPL" {
-        add_field => { "rule_type" => "Emerging Threats" }
+        mutate {
-        lowercase => [ "category"]
+          add_field => [ "signature_info", "https://www.snort.org/search?query=%{[alert][gid]}-%{[alert][signature_id]}" ]
+        }
+      }
+      if [rule_type] == "Emerging Threats" {
+        mutate {
+          add_field => [ "signature_info", "http://doc.emergingthreats.net/%{[alert][signature_id]}" ]
+        }
       }
     }
-	# I recommend changing the field types below to integer so searches can do greater than or less than
-	# and also so math functions can be ran against them
     mutate {
-      convert => [ "source_port", "integer" ]
+      remove_field => [ "alert" ]
-      convert => [ "destination_port", "integer" ]
-      convert => [ "gid", "integer" ]
-      convert => [ "sid", "integer" ]
-    #  remove_field => [ "message"]
     }
-	# This will translate the priority field into a severity field of either High, Medium, or Low
-	if [priority] == 1 {
-      mutate {
-        add_field => { "severity" => "High" }
-      }
-    }
-    if [priority] == 2 {
-      mutate {
-        add_field => { "severity" => "Medium" }
-      }
-    }
-    if [priority] == 3 {
-      mutate {
-        add_field => { "severity" => "Low" }
-      }
-    }
-    # This section adds URLs to lookup information about a rule online
-    if [sid] and [sid] > 0 and [sid] < 1000000 {
-      mutate {
-        add_field => [ "signature_info", "https://www.snort.org/search?query=%{gid}-%{sid}" ]
-      }
-    }
-    if [sid] and [sid] > 1999999 and [sid] < 2999999 {
-      mutate {
-        add_field => [ "signature_info", "http://doc.emergingthreats.net/%{sid}" ]
-      }
-    }
-#	mutate {
-		#add_tag => [ "conf_file_1033"]
-#	}
   }
 }

salt/logstash/pipelines/templates/so/logstash-template.json

@@ -738,6 +738,10 @@
             }
           }
         },
+	"destination":{
+          "type":"object",
+          "dynamic": true
+        },
         "destination_city":{
           "type":"text",
           "fields":{
@@ -2946,6 +2950,10 @@
             }
           }
         },
+	"source":{
+          "type":"object",
+          "dynamic": true
+        },
         "source_geo.city_name":{
           "type":"text",
           "fields":{

salt/strelka/init.sls

@@ -54,36 +54,8 @@ strelkastagedir:
     - group: 939
     - makedirs: True
 
-
-so-strelka-frontendimage:
- cmd.run:
-   - name: docker pull --disable-content-trust=false docker.io/soshybridhunter/so-strelka-frontend:HH1.1.5
-
-so-strelka-coordinatorimage:
- cmd.run:
-   - name: docker pull --disable-content-trust=false docker.io/redis:5.0.5-alpine3.10
-
-so-strelka-gatekeeperimage:
- cmd.run:
-   - name: docker pull --disable-content-trust=false docker.io/redis:5.0.5-alpine3.10
-
-so-strelka-backendimage:
- cmd.run:
-   - name: docker pull --disable-content-trust=false docker.io/soshybridhunter/so-strelka-backend:HH1.1.5
-
-so-strelka-managerimage:
- cmd.run:
-   - name: docker pull --disable-content-trust=false docker.io/soshybridhunter/so-strelka-manager:HH1.1.5
-
-so-strelka-backendimage:
- cmd.run:
-   - name: docker pull --disable-content-trust=false docker.io/soshybridhunter/so-strelka-backend:HH1.1.5
-
-
 strelka_coordinator:
   docker_container.running:
-    - require:
-      - so-strelka-coordinatorimage
     - image: docker.io/redis:5.0.5-alpine3.10
     - name: so-strelka-coordinator
     - command: redis-server --save "" --appendonly no
@@ -92,19 +64,15 @@ strelka_coordinator:
 
 strelka_gatekeeper:
   docker_container.running:
-    - require:
-      - so-strelka-gatekeeperimage
     - image: docker.io/redis:5.0.5-alpine3.10
     - name: so-strelka-gatekeeper
     - command: redis-server --save "" --appendonly no --maxmemory-policy allkeys-lru
     - port_bindings:
       - 0.0.0.0:6381:6379
-   
+
 strelka_frontend:
   docker_container.running:
-    - require:
+    - image: soshybridhunter/so-strelka-frontend:HH1.2.1
-      - so-strelka-frontendimage
-    - image: docker.io/soshybridhunter/so-strelka-frontend:HH1.1.5 
     - binds:
       - /opt/so/conf/strelka/frontend/:/etc/strelka/:ro
       - /opt/so/log/strelka/:/var/log/strelka/:rw
@@ -116,9 +84,7 @@ strelka_frontend:
 
 strelka_backend:
   docker_container.running:
-    - require:
+    - image: soshybridhunter/so-strelka-backend:HH1.2.1
-      - so-strelka-backendimage
-    - image: docker.io/soshybridhunter/so-strelka-backend:HH1.1.5
     - restart_policy: unless-stopped
     - binds:
       - /opt/so/conf/strelka/backend/:/etc/strelka/:ro
@@ -128,9 +94,7 @@ strelka_backend:
 
 strelka_manager:
   docker_container.running:
-    - require:
+    - image: soshybridhunter/so-strelka-manager:HH1.2.1
-      - so-strelka-managerimage
-    - image: docker.io/soshybridhunter/so-strelka-manager:HH1.1.5
     - binds:
       - /opt/so/conf/strelka/manager/:/etc/strelka/:ro
     - name: so-strelka-manager
@@ -138,12 +102,9 @@ strelka_manager:
 
 strelka_filestream:
   docker_container.running:
-    - require:
+    - image: soshybridhunter/so-strelka-filestream:HH1.2.1
-      - so-strelka-filestreamimage
-    - image: docker.io/soshybridhunter/so-strelka-filestream:HH1.1.5   
-    - image: docker.io/wlambert/sfilestream:grpc
     - binds:
       - /opt/so/conf/strelka/filestream/:/etc/strelka/:ro
       - /nsm/strelka:/nsm/strelka
     - name: so-strelka-filestream
-    - command: strelka-filestream
+    - command: strelka-filestream 

salt/zeek/files/local.zeek

@@ -124,3 +124,6 @@ redef LogAscii::json_timestamps = JSON::TS_ISO8601;
 
 # BPF Configuration
 @load securityonion/bpfconf
+
+# Extracted files
+@load securityonion/file-extraction

salt/zeek/policy/securityonion/file-extraction/extract.zeek

@@ -16,6 +16,6 @@ event file_sniff(f: fa_file, meta: fa_metadata)
     if ( meta?$mime_type )
         ext = ext_map[meta$mime_type];
 
-    local fname = fmt("/nsm/bro/extracted/%s-%s.%s", f$source, f$id, ext);
+    local fname = fmt("/nsm/zeek/extracted/%s-%s.%s", f$source, f$id, ext);
     Files::add_analyzer(f, Files::ANALYZER_EXTRACT, [$extract_filename=fname]);
     }