Merge pull request #428 from Security-Onion-Solutions/more_ecs

More ecs

salt/elasticsearch/files/ingest/common

@@ -39,7 +39,7 @@
     { "rename":         { "field": "dataset",              "target_field": "event.dataset",                  "ignore_missing": true  } },
     { 
 	"remove": {
-		"field": [ "index_name_prefix", "beat"],
+		"field": [ "index_name_prefix"],
 		"ignore_failure": false
 	}
     }

salt/elasticsearch/files/ingest/ossec.alert

@@ -2,6 +2,7 @@
   "description" : "ossec",
   "processors" : [
     { "json":		{ "field": "message",					"target_field": "message2",		"ignore_failure": true	} },
+    { "remove": { "field": [ "agent" ], "ignore_missing": true, "ignore_failure": false } },
     { "rename": 	{ "field": "message2.agent", 				"target_field": "agent",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.data", 				"target_field": "data",			"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.decoder", 				"target_field": "decoder",		"ignore_missing": true 	} },

salt/elasticsearch/files/ingest/strelka

@@ -6,7 +6,7 @@
     { "rename":         { "field": "message2.scan",        "target_field": "scan",          "ignore_missing": true  } },
     { "rename":         { "field": "message2.request",        "target_field": "request",          "ignore_missing": true  } },
     { "rename":         { "field": "scan.hash",        "target_field": "file.hash",          "ignore_missing": true  } },
-    { "remove":         { "field": ["host", "path"],                                         "ignore_missing": true  } },
+    { "remove":         { "field": ["host", "path", "agent"],                                         "ignore_missing": true  } },
     { "pipeline":       { "name": "common"                                                                                   } }
   ]
 }

salt/elasticsearch/files/ingest/suricata.common

@@ -8,7 +8,7 @@
     { "rename":{ "field": "message2.src_port",       "target_field": "source.port",  "ignore_failure": true  } },
     { "rename":{ "field": "message2.dest_ip",       "target_field": "destination.ip",  "ignore_failure": true  } },
     { "rename":{ "field": "message2.dest_port",       "target_field": "destination.port",  "ignore_failure": true  } },
-     { "remove":         { "field": ["message2"],     "ignore_failure": true                                                                  } },
+     { "remove":         { "field": ["message2", "agent"],     "ignore_failure": true                                                                  } },
     { "pipeline": { "name": "common" } }
   ]
 }

salt/elasticsearch/files/ingest/zeek.common

@@ -16,7 +16,7 @@
     { "rename":         { "field": "message2.id.resp_p",        "target_field": "destination.port",     "ignore_missing": true  } },
     { "set":         { "field": "server.port",        "value": "{{destination.port}}"  } },
     { "date":		{ "field": "message2.ts",	"target_field": "@timestamp",	"formats": ["ISO8601", "UNIX"], "ignore_failure": true	} },
-    { "remove":		{ "field": ["message2.ts", "path"],	"ignore_failure": true									} },
+    { "remove":		{ "field": ["message2.ts", "path", "agent"],	"ignore_failure": true									} },
     { "pipeline":	{ "name": "common" 													} }
   ]
 }

salt/elasticsearch/init.sls

@@ -114,11 +114,12 @@ so-elasticsearch:
     - name: so-elasticsearch
     - user: elasticsearch
     - environment:
-      - bootstrap.memory_lock=true
+      - discovery.type=single-node
-      - cluster.name={{ esclustername }}
+      #- bootstrap.memory_lock=true
+      #- cluster.name={{ esclustername }}
       - ES_JAVA_OPTS=-Xms{{ esheap }} -Xmx{{ esheap }}
-      - http.host=0.0.0.0
+      #- http.host=0.0.0.0
-      - transport.host=127.0.0.1
+      #- transport.host=127.0.0.1
     - ulimits:
       - memlock=-1:-1
       - nofile=65536:65536

salt/filebeat/etc/filebeat.yml

@@ -71,7 +71,7 @@ filebeat.modules:
 #=========================== Filebeat prospectors =============================
 
 # List of prospectors to fetch data.
-filebeat.prospectors:
+filebeat.inputs:
 #------------------------------ Log prospector --------------------------------
 {%- if grains['role'] == 'so-sensor' or grains['role'] == "so-eval" or grains['role'] == "so-helix" or grains['role'] == "so-heavynode" %}
 {%- if BROVER != 'SURICATA' %}

salt/logstash/pipelines/config/so/9001_output_switch.conf.jinja

@@ -21,7 +21,7 @@ output {
     elasticsearch {
       hosts => "{{ ES }}"
       index => "so-switch-%{+YYYY.MM.dd}"
-      template => "/so-template.json"
+      template => "/so-common-template.json"
     }
   }
 }

salt/logstash/pipelines/config/so/9002_output_import.conf.jinja

@@ -20,7 +20,7 @@ output {
       hosts => "{{ ES }}"
       index => "so-import-%{+YYYY.MM.dd}"
       template_name => "logstash"
-      template => "/so-template.json"
+      template => "/so-common-template.json"
       template_overwrite => true
     }
   }

salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja

@@ -21,7 +21,7 @@ output {
     elasticsearch {
       hosts => "{{ ES }}"
       index => "so-flow-%{+YYYY.MM.dd}"
-      template => "/so-template.json"
+      template => "/so-common-template.json"
     }
   }
 }

salt/logstash/pipelines/config/so/9026_output_dhcp.conf.jinja

@@ -20,7 +20,7 @@ output {
     #stdout { codec => rubydebug }
     elasticsearch {
       hosts => "{{ ES }}"
-      template => "/so-template.json"
+      template => "/so-common-template.json"
     }
   }
 }

salt/logstash/pipelines/config/so/9029_output_esxi.conf.jinja

@@ -19,7 +19,7 @@ output {
   if [event_type] == "esxi" and "test_data" not in [tags] {
     elasticsearch {
       hosts => "{{ ES }}"
-      template => "/so-template.json"
+      template => "/so-common-template.json"
     }
   }
 }

salt/logstash/pipelines/config/so/9030_output_greensql.conf.jinja

@@ -19,7 +19,7 @@ output {
   if [event_type] == "greensql" and "test_data" not in [tags] {
     elasticsearch {
       hosts => "{{ ES }}"
-      template => "/so-template.json"
+      template => "/so-common-template.json"
     }
   }
 }

salt/logstash/pipelines/config/so/9031_output_iis.conf.jinja

@@ -20,7 +20,7 @@ output {
     #stdout { codec => rubydebug }
     elasticsearch {
       hosts => "{{ ES }}"
-      template => "/so-template.json"
+      template => "/so-common-template.json"
     }
   }
 }

salt/logstash/pipelines/config/so/9032_output_mcafee.conf.jinja

@@ -20,7 +20,7 @@ output {
     #stdout { codec => rubydebug }
     elasticsearch {
       hosts => "{{ ES }}"
-      template => "/so-template.json"
+      template => "/so-common-template.json"
     }
   }
 }

salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja

@@ -22,7 +22,7 @@ output {
       hosts => "{{ ES }}"
       index => "so-ids-%{+YYYY.MM.dd}"
       template_name => "logstash"
-      template => "/so-template.json"
+      template => "/so-common-template.json"
       template_overwrite => true
     }
   }

salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja

@@ -21,7 +21,7 @@ output {
       hosts => "{{ ES }}"
       index => "so-syslog-%{+YYYY.MM.dd}"
       template_name => "logstash"
-      template => "/so-template.json"
+      template => "/so-common-template.json"
       template_overwrite => true
     }
   }

salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja

@@ -13,7 +13,7 @@ output {
     elasticsearch {
       hosts => "{{ ES }}"
       index => "so-osquery-%{+YYYY.MM.dd}"
-      template => "/so-template.json"
+      template => "/so-common-template.json"
     }
   }
 }

salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja

@@ -22,7 +22,7 @@ output {
       hosts => "{{ ES }}"
       index => "so-firewall-%{+YYYY.MM.dd}"
       template_name => "logstash"
-      template => "/so-template.json"
+      template => "/so-common-template.json"
       template_overwrite => true
     }
   }

salt/logstash/pipelines/config/so/9300_output_windows.conf.jinja

@@ -21,7 +21,7 @@ output {
     elasticsearch {
       hosts => "{{ ES }}"
       index => "so-windows-%{+YYYY.MM.dd}"
-      template => "/so-template.json"
+      template => "/so-common-template.json"
     }
   }
 }

salt/logstash/pipelines/config/so/9301_output_dns_windows.conf.jinja

@@ -21,7 +21,7 @@ output {
     elasticsearch {
       hosts => "{{ ES }}"
       index => "so-%{+YYYY.MM.dd}"
-      template => "/so-template.json"
+      template => "/so-common-template.json"
     }
   }
 }

salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja

@@ -21,7 +21,7 @@ output {
     elasticsearch {
       hosts => "{{ ES }}"
       index => "so-ids-%{+YYYY.MM.dd}"
-      template => "/so-template.json"
+      template => "/so-common-template.json"
     }
   }
 }