Merge pull request #975 from Security-Onion-Solutions/fix/lstemplate

Fix/lstemplate

pillar/logstash/eval.sls

@@ -19,6 +19,7 @@ logstash:
     - so/so-beats-template.json.jinja
     - so/so-common-template.json
     - so/so-firewall-template.json.jinja
+    - so/so-flow-template.json.jinja
     - so/so-ids-template.json.jinja
     - so/so-import-template.json.jinja
     - so/so-osquery-template.json.jinja

pillar/logstash/search.sls

@@ -15,6 +15,7 @@ logstash:
     - so/so-beats-template.json.jinja
     - so/so-common-template.json
     - so/so-firewall-template.json.jinja
+    - so/so-flow-template.json.jinja
     - so/so-ids-template.json.jinja
     - so/so-import-template.json.jinja
     - so/so-osquery-template.json.jinja

salt/filebeat/etc/filebeat.yml

@@ -126,6 +126,8 @@ filebeat.inputs:
       category: network
       imported: true
   processors:
+  - add_tags:
+      tags: [import]
   - dissect:
       tokenizer: "/nsm/import/%{import.id}/zeek/logs/%{import.file}"
       field: "log.file.path"
@@ -164,6 +166,8 @@ filebeat.inputs:
     category: network
     imported: true 
   processors:
+  - add_tags:
+      tags: [import]
   - dissect:
       tokenizer: "/nsm/import/%{import.id}/suricata/%{import.file}"
       field: "log.file.path"

salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja

@@ -3,21 +3,8 @@
 {%- else %}
 {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
 {%- endif %}
-# Author: Justin Henderson
-#         SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
-# Email: justin@hasecuritysolution.com
-# Last Update: 12/9/2016
-
-
-filter {
-  if [module] =~ "zeek" {
-    mutate {
-	  ##add_tag => [ "conf_file_9000"]
-	}
-  }
-}
 output {
-  if [module] =~ "zeek" {
+  if [module] =~ "zeek" and "import" not in [tags] {
     elasticsearch {
       pipeline => "%{module}.%{dataset}"
       hosts => "{{ ES }}"

salt/logstash/pipelines/config/so/9001_output_switch.conf.jinja

@@ -1,27 +0,0 @@
-{%- if grains['role'] == 'so-eval' -%}
-{%- set ES = salt['pillar.get']('master:mainip', '') -%}
-{%- else %}
-{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
-{%- endif %}
-# Author: Justin Henderson
-#         SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
-# Email: justin@hasecuritysolution.com
-# Last Update: 12/9/2016
-
-filter {
-  if "switch" in [tags] and "test_data" not in [tags] {
-    mutate {
-	  ##add_tag => [ "conf_file_9001"]
-	}
-  }
-}
-output {
-  if "switch" in [tags] and "test_data" not in [tags] {
-    #stdout { codec => rubydebug }
-    elasticsearch {
-      hosts => "{{ ES }}"
-      index => "so-switch-%{+YYYY.MM.dd}"
-      template => "/so-common-template.json"
-    }
-  }
-}

salt/logstash/pipelines/config/so/9002_output_import.conf.jinja

@@ -3,24 +3,14 @@
 {%- else %}
 {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
 {%- endif %}
-# Updated by: Doug Burks
-# Last Update: 5/16/2017
-
-filter {
-  if "import" in [tags] and "test_data" not in [tags] {
-    mutate {
-	  ##add_tag => [ "conf_file_9002"]
-	}
-  }
-}
 output {
-  if "import" in [tags] and "test_data" not in [tags] {
+  if "import" in [tags] {
-#    stdout { codec => rubydebug }
     elasticsearch {
+      pipeline => "%{module}.%{dataset}"
       hosts => "{{ ES }}"
       index => "so-import-%{+YYYY.MM.dd}"
-      template_name => "so-common"
+      template_name => "so-import"
-      template => "/so-common-template.json"
+      template => "/so-import-template.json"
       template_overwrite => true
     }
   }

salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja

@@ -3,25 +3,14 @@
 {%- else %}
 {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
 {%- endif %}
-# Author: Justin Henderson
-#         SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
-# Email: justin@hasecuritysolution.com
-# Last Update: 12/9/2016
-
-filter {
-  if [event_type] == "sflow" and "test_data" not in [tags] {
-    mutate {
-	  ##add_tag => [ "conf_file_9004"]
-	}
-  }
-}
 output {
-  if [event_type] == "sflow" and "test_data" not in [tags] {
+  if [event_type] == "sflow" {
-    #stdout { codec => rubydebug }
     elasticsearch {
       hosts => "{{ ES }}"
       index => "so-flow-%{+YYYY.MM.dd}"
-      template => "/so-common-template.json"
+      template_name => "so-flow"
+      template => "/so-flow-template.json"
+      template_overwrite => true
     }
   }
 }

salt/logstash/pipelines/config/so/9026_output_dhcp.conf.jinja

@@ -1,26 +0,0 @@
-{%- if grains['role'] == 'so-eval' -%}
-{%- set ES = salt['pillar.get']('master:mainip', '') -%}
-{%- else %}
-{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
-{%- endif %}
-# Author: Justin Henderson
-#         SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
-# Email: justin@hasecuritysolution.com
-# Last Update: 12/9/2016
-
-filter {
-  if [event_type] == "dhcp" and "test_data" not in [tags] {
-    mutate {
-	  ##add_tag => [ "conf_file_9026"]
-	}
-  }
-}
-output {
-  if [event_type] == "dhcp" and "test_data" not in [tags] {
-    #stdout { codec => rubydebug }
-    elasticsearch {
-      hosts => "{{ ES }}"
-      template => "/so-common-template.json"
-    }
-  }
-}

salt/logstash/pipelines/config/so/9029_output_esxi.conf.jinja

@@ -1,25 +0,0 @@
-{%- if grains['role'] == 'so-eval' -%}
-{%- set ES = salt['pillar.get']('master:mainip', '') -%}
-{%- else %}
-{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
-{%- endif %}
-# Author: Justin Henderson
-#         SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
-# Email: justin@hasecuritysolution.com
-# Last Update: 12/9/2016
-
-filter {
-  if [event_type] == "esxi" and "test_data" not in [tags] {
-    mutate {
-	  ##add_tag => [ "conf_file_9029"]
-	}
-  }
-}
-output {
-  if [event_type] == "esxi" and "test_data" not in [tags] {
-    elasticsearch {
-      hosts => "{{ ES }}"
-      template => "/so-common-template.json"
-    }
-  }
-}

salt/logstash/pipelines/config/so/9030_output_greensql.conf.jinja

@@ -1,25 +0,0 @@
-{%- if grains['role'] == 'so-eval' -%}
-{%- set ES = salt['pillar.get']('master:mainip', '') -%}
-{%- else %}
-{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
-{%- endif %}
-# Author: Justin Henderson
-#         SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
-# Email: justin@hasecuritysolution.com
-# Last Update: 12/9/2016
-
-filter {
-  if [event_type] == "greensql" and "test_data" not in [tags] {
-    mutate {
-	  ##add_tag => [ "conf_file_9030"]
-	}
-  }
-}
-output {
-  if [event_type] == "greensql" and "test_data" not in [tags] {
-    elasticsearch {
-      hosts => "{{ ES }}"
-      template => "/so-common-template.json"
-    }
-  }
-}

salt/logstash/pipelines/config/so/9031_output_iis.conf.jinja

@@ -1,26 +0,0 @@
-{%- if grains['role'] == 'so-eval' -%}
-{%- set ES = salt['pillar.get']('master:mainip', '') -%}
-{%- else %}
-{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
-{%- endif %}
-# Author: Justin Henderson
-#         SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
-# Email: justin@hasecuritysolution.com
-# Last Update: 12/9/2016
-
-filter {
-  if [event_type] == "iis" and "test_data" not in [tags] {
-    mutate {
-	  ##add_tag => [ "conf_file_9031"]
-	}
-  }
-}
-output {
-  if [event_type] == "iis" and "test_data" not in [tags] {
-    #stdout { codec => rubydebug }
-    elasticsearch {
-      hosts => "{{ ES }}"
-      template => "/so-common-template.json"
-    }
-  }
-}

salt/logstash/pipelines/config/so/9032_output_mcafee.conf.jinja

@@ -1,26 +0,0 @@
-{%- if grains['role'] == 'so-eval' -%}
-{%- set ES = salt['pillar.get']('master:mainip', '') -%}
-{%- else %}
-{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
-{%- endif %}
-# Author: Justin Henderson
-#         SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
-# Email: justin@hasecuritysolution.com
-# Last Update: 12/9/2016
-
-filter {
-  if [event_type] == "mcafee" and "test_data" not in [tags] {
-    mutate {
-	  ##add_tag => [ "conf_file_9032"]
-	}
-  }
-}
-output {
-  if [event_type] == "mcafee" and "test_data" not in [tags] {
-    #stdout { codec => rubydebug }
-    elasticsearch {
-      hosts => "{{ ES }}"
-      template => "/so-common-template.json"
-    }
-  }
-}

salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja

@@ -3,26 +3,13 @@
 {%- else %}
 {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
 {%- endif %}
-# Author: Justin Henderson
-#         SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
-# Email: justin@hasecuritysolution.com
-# Last Update: 12/9/2016
-
-filter {
-  if [event_type] == "ids" and "test_data" not in [tags] {
-    mutate {
-	  ##add_tag => [ "conf_file_9033"]
-	}
-  }
-}
 output {
-    if [event_type] == "ids" and "test_data" not in [tags] {
+  if [event_type] == "ids" and "import" not in [tags] {
-    #stdout { codec => rubydebug }
     elasticsearch {
       hosts => "{{ ES }}"
       index => "so-ids-%{+YYYY.MM.dd}"
-      template_name => "so-common"
+      template_name => "so-ids"
-      template => "/so-common-template.json"
+      template => "/so-ids-template.json"
       template_overwrite => true
     }
   }

salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja

@@ -3,22 +3,14 @@
 {%- else %}
 {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
 {%- endif %}
-
-filter {
-  if [module] =~ "syslog" {
-    mutate {
-          ##add_tag => [ "conf_file_9000"]
-        }
-  }
-}
 output {
   if [module] =~ "syslog" {
     elasticsearch {
       pipeline => "%{module}"
       hosts => "{{ ES }}"
       index => "so-syslog-%{+YYYY.MM.dd}"
-      template_name => "so-common"
+      template_name => "so-syslog"
-      template => "/so-common-template.json"
+      template => "/so-syslog-template.json"
       template_overwrite => true
     }
   }

salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja

@@ -3,18 +3,15 @@
 {%- else %}
 {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
 {%- endif %}
-# Author: Josh Brower
-# Last Update: 12/29/2018
-# Output to ES for osquery tagged logs
-
-
 output {
   if [module] =~ "osquery" {
     elasticsearch {
       pipeline => "%{module}.%{dataset}" 
       hosts => "{{ ES }}"
       index => "so-osquery-%{+YYYY.MM.dd}"
-      template => "/so-common-template.json"
+      template_name => "so-osquery"
+      template => "/so-osquery-template.json"
+      template_overwrite => true
     }
   }
 }

salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja

@@ -3,26 +3,13 @@
 {%- else %}
 {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
 {%- endif %}
-# Author: Justin Henderson
-#         SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
-# Email: justin@hasecuritysolution.com
-# Last Update: 12/9/2016
-
-filter {
-  if "firewall" in [tags] and "test_data" not in [tags] {
-    mutate {
-	  ##add_tag => [ "conf_file_9200"]
-	}
-  }
-}
 output {
-  if "firewall" in [tags] and "test_data" not in [tags] {
+  if "firewall" in [tags] {
-#    stdout { codec => rubydebug }
     elasticsearch {
       hosts => "{{ ES }}"
       index => "so-firewall-%{+YYYY.MM.dd}"
-      template_name => "so-common"
+      template_name => "so-firewall"
-      template => "/so-common-template.json"
+      template => "/so-firewall-template.json"
       template_overwrite => true
     }
   }

salt/logstash/pipelines/config/so/9300_output_windows.conf.jinja

@@ -1,27 +0,0 @@
-{%- if grains['role'] == 'so-eval' -%}
-{%- set ES = salt['pillar.get']('master:mainip', '') -%}
-{%- else %}
-{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
-{%- endif %}
-# Author: Justin Henderson
-#         SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
-# Email: justin@hasecuritysolution.com
-# Last Update: 12/9/2016
-
-filter {
-  if [event_type] == "windows" and "test_data" not in [tags] {
-    mutate {
-	  ##add_tag => [ "conf_file_9300"]
-	}
-  }
-}
-output {
-  if [event_type] == "windows" and "test_data" not in [tags] {
-    #stdout { codec => rubydebug }
-    elasticsearch {
-      hosts => "{{ ES }}"
-      index => "so-windows-%{+YYYY.MM.dd}"
-      template => "/so-common-template.json"
-    }
-  }
-}

salt/logstash/pipelines/config/so/9301_output_dns_windows.conf.jinja

@@ -1,27 +0,0 @@
-{%- if grains['role'] == 'so-eval' -%}
-{%- set ES = salt['pillar.get']('master:mainip', '') -%}
-{%- else %}
-{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
-{%- endif %}
-# Author: Justin Henderson
-#         SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
-# Email: justin@hasecuritysolution.com
-# Last Update: 12/9/2016
-
-filter {
-  if [event_type] == "dns" and "test_data" not in [tags] {
-    mutate {
-	  ##add_tag => [ "conf_file_9301"]
-	}
-  }
-}
-output {
-  if [event_type] == "dns" and "test_data" not in [tags] {
-    #stdout { codec => rubydebug }
-    elasticsearch {
-      hosts => "{{ ES }}"
-      index => "so-%{+YYYY.MM.dd}"
-      template => "/so-common-template.json"
-    }
-  }
-}

salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja

@@ -3,26 +3,14 @@
 {%- else %}
 {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
 {%- endif %}
-# Author: Justin Henderson
-#         SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
-# Email: justin@hasecuritysolution.com
-# Last Update: 12/9/2016
-
-filter {
-  if [module] == "suricata" {
-    mutate {
-	  ##add_tag => [ "conf_file_9400"]
-	}
-  }
-}
 output {
-  if [module] =~ "suricata" {
+  if [module] =~ "suricata" and "import" not in [tags] {
     elasticsearch {
       pipeline => "%{module}.%{dataset}"
       hosts => "{{ ES }}"
       index => "so-ids-%{+YYYY.MM.dd}"
-      template_name => "so-common" 
+      template_name => "so-ids" 
-      template => "/so-common-template.json"
+      template => "/so-ids-template.json"
     }
   }
 }

salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja

@@ -3,15 +3,14 @@
 {%- else %}
 {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
 {%- endif %}
-
 output {
-  if "beat-ext" in [tags] {
+  if "beat-ext" in [tags] and "import" not in [tags] {
     elasticsearch {
       pipeline => "beats.common" 
       hosts => "{{ ES }}"
       index => "so-beats-%{+YYYY.MM.dd}"
-      template_name => "so-common"
+      template_name => "so-beats"
-      template => "/so-common-template.json"
+      template => "/so-beats-template.json"
       template_overwrite => true
     }
   }

salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja

@@ -3,27 +3,14 @@
 {%- else %}
 {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
 {%- endif %}
-# Author: Justin Henderson
-#         SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
-# Updated by: Doug Burks
-# Last Update: 9/19/2018
-
-filter {
-  if [module] =~ "ossec" {
-    mutate {
-          ##add_tag => [ "conf_file_9600"]
-        }
-  }
-}
-
 output {
   if [module] =~ "ossec" {
     elasticsearch {
       pipeline => "%{module}.%{dataset}"
       hosts => "{{ ES }}"
       index => "so-ossec-%{+YYYY.MM.dd}"
-      template_name => "so-common"
+      template_name => "so-ossec"
-      template => "/so-common-template.json"
+      template => "/so-ossec-template.json"
       template_overwrite => true
     }
   }

salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja

@@ -3,27 +3,14 @@
 {%- else %}
 {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
 {%- endif %}
-# Author: Justin Henderson
-#         SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
-# Email: justin@hasecuritysolution.com
-# Last Update: 12/9/2016
-
-
-filter {
-  if [module] =~ "strelka" {
-    mutate {
-          ##add_tag => [ "conf_file_9000"]
-        }
-  }
-}
 output {
   if [event_type] =~ "strelka" {
     elasticsearch {
       pipeline => "%{module}.%{dataset}" 
       hosts => "{{ ES }}"
       index => "so-strelka-%{+YYYY.MM.dd}"
-      template_name => "so-common"
+      template_name => "so-strelka"
-      template => "/so-common-template.json"
+      template => "/so-strelka-template.json"
       template_overwrite => true
     }
   }

salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja

@@ -1,6 +1,5 @@
 {% set MASTER = salt['pillar.get']('static:masterip', '') %}
 {% set BATCH = salt['pillar.get']('logstash_settings:ls_pipeline_batch_size', 125) %}
-
 output {
 	redis {
 		host => '{{ MASTER }}'

salt/logstash/pipelines/templates/so/so-flow-template.json.jinja

@@ -0,0 +1,13 @@
+{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-flow:shards', 1) %}
+{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %}
+{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-flow:refresh', '30s') %}
+{
+  "index_patterns": ["so-flow-*"],
+  "version": 50001,
+  "order": 11,
+  "settings":{
+    "number_of_replicas":{{ REPLICAS }},
+    "number_of_shards":{{ SHARDS }},
+    "index.refresh_interval":"{{ REFRESH }}"
+  }
+}

setup/so-functions

@@ -1026,6 +1026,11 @@ master_static() {
 		"      warm: 7"\
 		"      close: 30"\
 		"      delete: 365"\
+		"    so-flow:"\
+		"      shards: 1"\
+		"      warm: 7"\
+		"      close: 30"\
+		"      delete: 365"\
 		"    so-ids:"\
 		"      shards: 1"\
 		"      warm: 7"\