merge with dev and resolve conflicts

pillar/logstash/eval.sls

@@ -19,6 +19,7 @@ logstash:
     - so/so-beats-template.json.jinja
     - so/so-common-template.json
     - so/so-firewall-template.json.jinja
+    - so/so-flow-template.json.jinja
     - so/so-ids-template.json.jinja
     - so/so-import-template.json.jinja
     - so/so-osquery-template.json.jinja

pillar/logstash/search.sls

@@ -15,6 +15,7 @@ logstash:
     - so/so-beats-template.json.jinja
     - so/so-common-template.json
     - so/so-firewall-template.json.jinja
+    - so/so-flow-template.json.jinja
     - so/so-ids-template.json.jinja
     - so/so-import-template.json.jinja
     - so/so-osquery-template.json.jinja

salt/common/init.sls

@@ -1,3 +1,5 @@
+{% set role = grains.id.split('_') | last %}
+
 # Add socore Group
 socoregroup:
   group.present:
@@ -131,3 +133,15 @@ utilsyncscripts:
     - file_mode: 755
     - template: jinja
     - source: salt://common/tools/sbin
+
+{% if role in ['eval', 'standalone', 'sensor', 'heavynode'] %}
+# Add sensor cleanup
+/usr/sbin/so-sensor-clean:
+  cron.present:
+    - user: root
+    - minute: '*'
+    - hour: '*'
+    - daymonth: '*'
+    - month: '*'
+    - dayweek: '*'
+{% endif %}

salt/common/tools/sbin/so-import-pcap

@@ -23,7 +23,8 @@ function usage {
   cat << EOF
 Usage: $0 <pcap-file-1> [pcap-file-2] [pcap-file-N]
 
-Imports one or more PCAP files for analysis. If available, curator will be automatically stopped.
+Imports one or more PCAP files onto a sensor node. The PCAP traffic will be analyzed and
+made available for review in the Security Onion toolset.
 EOF
 }
 
@@ -110,14 +111,6 @@ for i in "$@"; do
   fi
 done
 
-if ! [ -d /opt/so/conf/curator ]; then
-  echo "Curator is not installed on this node and cannot be stopped automatically."
-else
-  echo -n "Stopping curator..."
-  so-curator-stop > /dev/null 2>&1
-  echo "Done"
-fi
-
 # track if we have any valid or invalid pcaps
 INVALID_PCAPS="no"
 VALID_PCAPS="no"
@@ -220,6 +213,6 @@ https://{{ MANAGERIP }}/kibana/app/kibana#/dashboard/a8411b30-6d03-11ea-b301-3d6
 or you can manually set your Time Range to be:
 From: $START_OLDEST    To: $END_NEWEST
 
-Please note that it may take 30 seconds or more for events to appear in Kibana.
+Please note that it may take 30 seconds or more for events to appear in Onion Hunt.
 EOF
 fi

salt/common/tools/sbin/so-sensor-clean

@@ -0,0 +1,121 @@
+#!/bin/bash
+
+# Delete Zeek Logs based on defined CRIT_DISK_USAGE value
+
+# Copyright 2014,2015,2016,2017,2018, 2019 Security Onion Solutions, LLC
+
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+SENSOR_DIR='/nsm'
+CRIT_DISK_USAGE=30
+CUR_USAGE=$(df -P $SENSOR_DIR | tail -1 | awk '{print $5}' | tr -d %)
+LOG="/opt/so/log/sensor_clean.log"
+TODAY=$(date -u "+%Y-%m-%d")
+
+clean () {
+                ## find the oldest Zeek logs directory
+                OLDEST_DIR=$(ls /nsm/zeek/logs/ | grep -v "current" | grep -v "stats" | grep -v "packetloss" | grep -v "zeek_clean" | sort | head -n 1)
+                if [ -z "$OLDEST_DIR" -o "$OLDEST_DIR" == ".." -o "$OLDEST_DIR" == "." ]
+                then
+                        echo "$(date) - No old Zeek logs available to clean up in /nsm/zeek/logs/" >> $LOG
+                        #exit 0
+                else
+                        echo "$(date) - Removing directory: /nsm/zeek/logs/$OLDEST_DIR" >> $LOG
+                        rm -rf /nsm/zeek/logs/"$OLDEST_DIR"
+                fi
+
+
+                ## Remarking for now, as we are moving extracted files to /nsm/strelka/processed
+                ## find oldest files in extracted directory and exclude today
+                #OLDEST_EXTRACT=$(find /nsm/zeek/extracted/complete -type f -printf '%T+ %p\n' 2>/dev/null | sort | grep -v $TODAY | head -n 1)
+                #if [ -z "$OLDEST_EXTRACT" -o "$OLDEST_EXTRACT" == ".." -o "$OLDEST_EXTRACT" == "." ]
+                #then
+                #        echo "$(date) - No old extracted files available to clean up in /nsm/zeek/extracted/complete" >> $LOG
+                #else
+                #        OLDEST_EXTRACT_DATE=`echo $OLDEST_EXTRACT | awk '{print $1}' | cut -d+ -f1`
+                #        OLDEST_EXTRACT_FILE=`echo $OLDEST_EXTRACT | awk '{print $2}'`
+                #        echo "$(date) - Removing extracted files for $OLDEST_EXTRACT_DATE" >> $LOG
+                #        find /nsm/zeek/extracted/complete -type f -printf '%T+ %p\n' | grep $OLDEST_EXTRACT_DATE | awk '{print $2}' |while read FILE
+                #        do
+                #                echo "$(date) - Removing extracted file: $FILE" >> $LOG
+                #                rm -f "$FILE"
+                #        done
+                #fi
+ 
+                ## Clean up Zeek extracted files processed by Strelka
+                STRELKA_FILES='/nsm/strelka/processed'
+                OLDEST_STRELKA=$(find $STRELKA_FILES -type f -printf '%T+ %p\n' | sort -n | head -n 1 )
+                if [ -z "$OLDEST_STRELKA" -o "$OLDEST_STRELKA" == ".." -o "$OLDEST_STRELKA" == "." ]
+                then
+                        echo "$(date) - No old files available to clean up in $STRELKA_FILES" >> $LOG
+                else
+                        OLDEST_STRELKA_DATE=`echo $OLDEST_STRELKA | awk '{print $1}' | cut -d+ -f1`
+                        OLDEST_STRELKA_FILE=`echo $OLDEST_STRELKA | awk '{print $2}'`
+                        echo "$(date) - Removing extracted files for $OLDEST_STRELKA_DATE" >> $LOG
+                        find $STRELKA_FILES -type f -printf '%T+ %p\n' | grep $OLDEST_STRELKA_DATE | awk '{print $2}' |while read FILE
+                        do
+                                echo "$(date) - Removing file: $FILE" >> $LOG
+                                rm -f "$FILE"
+                        done
+                fi
+
+                ## Clean up Suricata log files
+                SURICATA_LOGS='/nsm/suricata'
+                OLDEST_SURICATA=$(find $STRELKA_FILES -type f -printf '%T+ %p\n' | sort -n | head -n 1)
+                if [ -z "$OLDEST_SURICATA" -o "$OLDEST_SURICATA" == ".." -o "$OLDEST_SURICATA" == "." ]
+                then
+                        echo "$(date) - No old files available to clean up in $SURICATA_LOGS" >> $LOG
+                else
+                        OLDEST_SURICATA_DATE=`echo $OLDEST_SURICATA | awk '{print $1}' | cut -d+ -f1`
+                        OLDEST_SURICATA_FILE=`echo $OLDEST_SURICATA | awk '{print $2}'`
+                        echo "$(date) - Removing logs for $OLDEST_SURICATA_DATE" >> $LOG
+                        find $SURICATA_LOGS -type f -printf '%T+ %p\n' | grep $OLDEST_SURICATA_DATE | awk '{print $2}' |while read FILE
+                        do
+                                echo "$(date) - Removing file: $FILE" >> $LOG
+                                rm -f "$FILE"
+                        done
+                fi
+
+                ## Clean up extracted pcaps from Steno
+                PCAPS='/nsm/pcapout'
+                OLDEST_PCAP=$(find $PCAPS -type f -printf '%T+ %p\n' | sort -n | head -n 1 )
+                if [ -z "$OLDEST_PCAP" -o "$OLDEST_PCAP" == ".." -o "$OLDEST_PCAP" == "." ]
+                then
+                        echo "$(date) - No old files available to clean up in $PCAPS" >> $LOG
+                else
+                        OLDEST_PCAP_DATE=`echo $OLDEST_PCAP | awk '{print $1}' | cut -d+ -f1`
+                        OLDEST_PCAP_FILE=`echo $OLDEST_PCAP | awk '{print $2}'`
+                        echo "$(date) - Removing extracted files for $OLDEST_PCAP_DATE" >> $LOG
+                        find $PCAPS -type f -printf '%T+ %p\n' | grep $OLDEST_PCAP_DATE | awk '{print $2}' |while read FILE
+                        do
+                                echo "$(date) - Removing file: $FILE" >> $LOG
+                                rm -f "$FILE"
+                        done
+                fi
+}
+
+# Check to see if we are already running
+IS_RUNNING=$(ps aux | grep "sensor_clean" | grep -v grep | wc -l)
+[ "$IS_RUNNING" -gt 2 ] && echo "$(date) - $IS_RUNNING sensor clean script processes running...exiting." >> $LOG && exit 0
+
+if [ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ]; then
+    while [ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ];
+        do
+         clean
+         CUR_USAGE=$(df -P $SENSOR_DIR | tail -1 | awk '{print $5}' | tr -d %)
+    done
+else
+    echo "$(date) - Current usage value of $CUR_USAGE not greater than CRIT_DISK_USAGE value of $CRIT_DISK_USAGE..." >> $LOG
+fi
+

salt/filebeat/etc/filebeat.yml

@@ -126,6 +126,8 @@ filebeat.inputs:
       category: network
       imported: true
   processors:
+  - add_tags:
+      tags: [import]
   - dissect:
       tokenizer: "/nsm/import/%{import.id}/zeek/logs/%{import.file}"
       field: "log.file.path"
@@ -164,6 +166,8 @@ filebeat.inputs:
     category: network
     imported: true 
   processors:
+  - add_tags:
+      tags: [import]
   - dissect:
       tokenizer: "/nsm/import/%{import.id}/suricata/%{import.file}"
       field: "log.file.path"

salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja

@@ -3,21 +3,8 @@
 {%- else %}
 {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
 {%- endif %}
-# Author: Justin Henderson
-#         SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
-# Email: justin@hasecuritysolution.com
-# Last Update: 12/9/2016
-
-
-filter {
-  if [module] =~ "zeek" {
-    mutate {
-	  ##add_tag => [ "conf_file_9000"]
-	}
-  }
-}
 output {
-  if [module] =~ "zeek" {
+  if [module] =~ "zeek" and "import" not in [tags] {
     elasticsearch {
       pipeline => "%{module}.%{dataset}"
       hosts => "{{ ES }}"

salt/logstash/pipelines/config/so/9001_output_switch.conf.jinja

@@ -1,27 +0,0 @@
-{%- if grains['role'] == 'so-eval' -%}
-{%- set ES = salt['pillar.get']('manager:mainip', '') -%}
-{%- else %}
-{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
-{%- endif %}
-# Author: Justin Henderson
-#         SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
-# Email: justin@hasecuritysolution.com
-# Last Update: 12/9/2016
-
-filter {
-  if "switch" in [tags] and "test_data" not in [tags] {
-    mutate {
-	  ##add_tag => [ "conf_file_9001"]
-	}
-  }
-}
-output {
-  if "switch" in [tags] and "test_data" not in [tags] {
-    #stdout { codec => rubydebug }
-    elasticsearch {
-      hosts => "{{ ES }}"
-      index => "so-switch-%{+YYYY.MM.dd}"
-      template => "/so-common-template.json"
-    }
-  }
-}

salt/logstash/pipelines/config/so/9002_output_import.conf.jinja

@@ -3,24 +3,14 @@
 {%- else %}
 {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
 {%- endif %}
-# Updated by: Doug Burks
-# Last Update: 5/16/2017
-
-filter {
-  if "import" in [tags] and "test_data" not in [tags] {
-    mutate {
-	  ##add_tag => [ "conf_file_9002"]
-	}
-  }
-}
 output {
-  if "import" in [tags] and "test_data" not in [tags] {
+  if "import" in [tags] {
-#    stdout { codec => rubydebug }
     elasticsearch {
+      pipeline => "%{module}.%{dataset}"
       hosts => "{{ ES }}"
       index => "so-import-%{+YYYY.MM.dd}"
-      template_name => "so-common"
+      template_name => "so-import"
-      template => "/so-common-template.json"
+      template => "/so-import-template.json"
       template_overwrite => true
     }
   }

salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja

@@ -3,25 +3,14 @@
 {%- else %}
 {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
 {%- endif %}
-# Author: Justin Henderson
-#         SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
-# Email: justin@hasecuritysolution.com
-# Last Update: 12/9/2016
-
-filter {
-  if [event_type] == "sflow" and "test_data" not in [tags] {
-    mutate {
-	  ##add_tag => [ "conf_file_9004"]
-	}
-  }
-}
 output {
-  if [event_type] == "sflow" and "test_data" not in [tags] {
+  if [event_type] == "sflow" {
-    #stdout { codec => rubydebug }
     elasticsearch {
       hosts => "{{ ES }}"
       index => "so-flow-%{+YYYY.MM.dd}"
-      template => "/so-common-template.json"
+      template_name => "so-flow"
+      template => "/so-flow-template.json"
+      template_overwrite => true
     }
   }
 }

salt/logstash/pipelines/config/so/9026_output_dhcp.conf.jinja

@@ -1,26 +0,0 @@
-{%- if grains['role'] == 'so-eval' -%}
-{%- set ES = salt['pillar.get']('manager:mainip', '') -%}
-{%- else %}
-{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
-{%- endif %}
-# Author: Justin Henderson
-#         SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
-# Email: justin@hasecuritysolution.com
-# Last Update: 12/9/2016
-
-filter {
-  if [event_type] == "dhcp" and "test_data" not in [tags] {
-    mutate {
-	  ##add_tag => [ "conf_file_9026"]
-	}
-  }
-}
-output {
-  if [event_type] == "dhcp" and "test_data" not in [tags] {
-    #stdout { codec => rubydebug }
-    elasticsearch {
-      hosts => "{{ ES }}"
-      template => "/so-common-template.json"
-    }
-  }
-}

salt/logstash/pipelines/config/so/9029_output_esxi.conf.jinja

@@ -1,25 +0,0 @@
-{%- if grains['role'] == 'so-eval' -%}
-{%- set ES = salt['pillar.get']('manager:mainip', '') -%}
-{%- else %}
-{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
-{%- endif %}
-# Author: Justin Henderson
-#         SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
-# Email: justin@hasecuritysolution.com
-# Last Update: 12/9/2016
-
-filter {
-  if [event_type] == "esxi" and "test_data" not in [tags] {
-    mutate {
-	  ##add_tag => [ "conf_file_9029"]
-	}
-  }
-}
-output {
-  if [event_type] == "esxi" and "test_data" not in [tags] {
-    elasticsearch {
-      hosts => "{{ ES }}"
-      template => "/so-common-template.json"
-    }
-  }
-}

salt/logstash/pipelines/config/so/9030_output_greensql.conf.jinja

@@ -1,25 +0,0 @@
-{%- if grains['role'] == 'so-eval' -%}
-{%- set ES = salt['pillar.get']('manager:mainip', '') -%}
-{%- else %}
-{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
-{%- endif %}
-# Author: Justin Henderson
-#         SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
-# Email: justin@hasecuritysolution.com
-# Last Update: 12/9/2016
-
-filter {
-  if [event_type] == "greensql" and "test_data" not in [tags] {
-    mutate {
-	  ##add_tag => [ "conf_file_9030"]
-	}
-  }
-}
-output {
-  if [event_type] == "greensql" and "test_data" not in [tags] {
-    elasticsearch {
-      hosts => "{{ ES }}"
-      template => "/so-common-template.json"
-    }
-  }
-}

salt/logstash/pipelines/config/so/9031_output_iis.conf.jinja

@@ -1,26 +0,0 @@
-{%- if grains['role'] == 'so-eval' -%}
-{%- set ES = salt['pillar.get']('manager:mainip', '') -%}
-{%- else %}
-{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
-{%- endif %}
-# Author: Justin Henderson
-#         SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
-# Email: justin@hasecuritysolution.com
-# Last Update: 12/9/2016
-
-filter {
-  if [event_type] == "iis" and "test_data" not in [tags] {
-    mutate {
-	  ##add_tag => [ "conf_file_9031"]
-	}
-  }
-}
-output {
-  if [event_type] == "iis" and "test_data" not in [tags] {
-    #stdout { codec => rubydebug }
-    elasticsearch {
-      hosts => "{{ ES }}"
-      template => "/so-common-template.json"
-    }
-  }
-}

salt/logstash/pipelines/config/so/9032_output_mcafee.conf.jinja

@@ -1,26 +0,0 @@
-{%- if grains['role'] == 'so-eval' -%}
-{%- set ES = salt['pillar.get']('manager:mainip', '') -%}
-{%- else %}
-{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
-{%- endif %}
-# Author: Justin Henderson
-#         SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
-# Email: justin@hasecuritysolution.com
-# Last Update: 12/9/2016
-
-filter {
-  if [event_type] == "mcafee" and "test_data" not in [tags] {
-    mutate {
-	  ##add_tag => [ "conf_file_9032"]
-	}
-  }
-}
-output {
-  if [event_type] == "mcafee" and "test_data" not in [tags] {
-    #stdout { codec => rubydebug }
-    elasticsearch {
-      hosts => "{{ ES }}"
-      template => "/so-common-template.json"
-    }
-  }
-}

salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja

@@ -3,26 +3,13 @@
 {%- else %}
 {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
 {%- endif %}
-# Author: Justin Henderson
-#         SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
-# Email: justin@hasecuritysolution.com
-# Last Update: 12/9/2016
-
-filter {
-  if [event_type] == "ids" and "test_data" not in [tags] {
-    mutate {
-	  ##add_tag => [ "conf_file_9033"]
-	}
-  }
-}
 output {
-    if [event_type] == "ids" and "test_data" not in [tags] {
+  if [event_type] == "ids" and "import" not in [tags] {
-    #stdout { codec => rubydebug }
     elasticsearch {
       hosts => "{{ ES }}"
       index => "so-ids-%{+YYYY.MM.dd}"
-      template_name => "so-common"
+      template_name => "so-ids"
-      template => "/so-common-template.json"
+      template => "/so-ids-template.json"
       template_overwrite => true
     }
   }

salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja

@@ -3,22 +3,14 @@
 {%- else %}
 {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
 {%- endif %}
-
-filter {
-  if [module] =~ "syslog" {
-    mutate {
-          ##add_tag => [ "conf_file_9000"]
-        }
-  }
-}
 output {
   if [module] =~ "syslog" {
     elasticsearch {
       pipeline => "%{module}"
       hosts => "{{ ES }}"
       index => "so-syslog-%{+YYYY.MM.dd}"
-      template_name => "so-common"
+      template_name => "so-syslog"
-      template => "/so-common-template.json"
+      template => "/so-syslog-template.json"
       template_overwrite => true
     }
   }

salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja

@@ -3,18 +3,15 @@
 {%- else %}
 {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
 {%- endif %}
-# Author: Josh Brower
-# Last Update: 12/29/2018
-# Output to ES for osquery tagged logs
-
-
 output {
   if [module] =~ "osquery" {
     elasticsearch {
       pipeline => "%{module}.%{dataset}" 
       hosts => "{{ ES }}"
       index => "so-osquery-%{+YYYY.MM.dd}"
-      template => "/so-common-template.json"
+      template_name => "so-osquery"
+      template => "/so-osquery-template.json"
+      template_overwrite => true
     }
   }
 }

salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja

@@ -3,26 +3,13 @@
 {%- else %}
 {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
 {%- endif %}
-# Author: Justin Henderson
-#         SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
-# Email: justin@hasecuritysolution.com
-# Last Update: 12/9/2016
-
-filter {
-  if "firewall" in [tags] and "test_data" not in [tags] {
-    mutate {
-	  ##add_tag => [ "conf_file_9200"]
-	}
-  }
-}
 output {
-  if "firewall" in [tags] and "test_data" not in [tags] {
+  if "firewall" in [tags] {
-#    stdout { codec => rubydebug }
     elasticsearch {
       hosts => "{{ ES }}"
       index => "so-firewall-%{+YYYY.MM.dd}"
-      template_name => "so-common"
+      template_name => "so-firewall"
-      template => "/so-common-template.json"
+      template => "/so-firewall-template.json"
       template_overwrite => true
     }
   }

salt/logstash/pipelines/config/so/9300_output_windows.conf.jinja

@@ -1,27 +0,0 @@
-{%- if grains['role'] == 'so-eval' -%}
-{%- set ES = salt['pillar.get']('manager:mainip', '') -%}
-{%- else %}
-{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
-{%- endif %}
-# Author: Justin Henderson
-#         SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
-# Email: justin@hasecuritysolution.com
-# Last Update: 12/9/2016
-
-filter {
-  if [event_type] == "windows" and "test_data" not in [tags] {
-    mutate {
-	  ##add_tag => [ "conf_file_9300"]
-	}
-  }
-}
-output {
-  if [event_type] == "windows" and "test_data" not in [tags] {
-    #stdout { codec => rubydebug }
-    elasticsearch {
-      hosts => "{{ ES }}"
-      index => "so-windows-%{+YYYY.MM.dd}"
-      template => "/so-common-template.json"
-    }
-  }
-}

salt/logstash/pipelines/config/so/9301_output_dns_windows.conf.jinja

@@ -1,27 +0,0 @@
-{%- if grains['role'] == 'so-eval' -%}
-{%- set ES = salt['pillar.get']('manager:mainip', '') -%}
-{%- else %}
-{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
-{%- endif %}
-# Author: Justin Henderson
-#         SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
-# Email: justin@hasecuritysolution.com
-# Last Update: 12/9/2016
-
-filter {
-  if [event_type] == "dns" and "test_data" not in [tags] {
-    mutate {
-	  ##add_tag => [ "conf_file_9301"]
-	}
-  }
-}
-output {
-  if [event_type] == "dns" and "test_data" not in [tags] {
-    #stdout { codec => rubydebug }
-    elasticsearch {
-      hosts => "{{ ES }}"
-      index => "so-%{+YYYY.MM.dd}"
-      template => "/so-common-template.json"
-    }
-  }
-}

salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja

@@ -3,26 +3,14 @@
 {%- else %}
 {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
 {%- endif %}
-# Author: Justin Henderson
-#         SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
-# Email: justin@hasecuritysolution.com
-# Last Update: 12/9/2016
-
-filter {
-  if [module] == "suricata" {
-    mutate {
-	  ##add_tag => [ "conf_file_9400"]
-	}
-  }
-}
 output {
-  if [module] =~ "suricata" {
+  if [module] =~ "suricata" and "import" not in [tags] {
     elasticsearch {
       pipeline => "%{module}.%{dataset}"
       hosts => "{{ ES }}"
       index => "so-ids-%{+YYYY.MM.dd}"
-      template_name => "so-common" 
+      template_name => "so-ids" 
-      template => "/so-common-template.json"
+      template => "/so-ids-template.json"
     }
   }
 }

salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja

@@ -3,15 +3,14 @@
 {%- else %}
 {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
 {%- endif %}
-
 output {
-  if "beat-ext" in [tags] {
+  if "beat-ext" in [tags] and "import" not in [tags] {
     elasticsearch {
       pipeline => "beats.common" 
       hosts => "{{ ES }}"
       index => "so-beats-%{+YYYY.MM.dd}"
-      template_name => "so-common"
+      template_name => "so-beats"
-      template => "/so-common-template.json"
+      template => "/so-beats-template.json"
       template_overwrite => true
     }
   }

salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja

@@ -3,27 +3,14 @@
 {%- else %}
 {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
 {%- endif %}
-# Author: Justin Henderson
-#         SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
-# Updated by: Doug Burks
-# Last Update: 9/19/2018
-
-filter {
-  if [module] =~ "ossec" {
-    mutate {
-          ##add_tag => [ "conf_file_9600"]
-        }
-  }
-}
-
 output {
   if [module] =~ "ossec" {
     elasticsearch {
       pipeline => "%{module}.%{dataset}"
       hosts => "{{ ES }}"
       index => "so-ossec-%{+YYYY.MM.dd}"
-      template_name => "so-common"
+      template_name => "so-ossec"
-      template => "/so-common-template.json"
+      template => "/so-ossec-template.json"
       template_overwrite => true
     }
   }

salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja

@@ -3,27 +3,14 @@
 {%- else %}
 {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
 {%- endif %}
-# Author: Justin Henderson
-#         SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
-# Email: justin@hasecuritysolution.com
-# Last Update: 12/9/2016
-
-
-filter {
-  if [module] =~ "strelka" {
-    mutate {
-          ##add_tag => [ "conf_file_9000"]
-        }
-  }
-}
 output {
   if [event_type] =~ "strelka" {
     elasticsearch {
       pipeline => "%{module}.%{dataset}" 
       hosts => "{{ ES }}"
       index => "so-strelka-%{+YYYY.MM.dd}"
-      template_name => "so-common"
+      template_name => "so-strelka"
-      template => "/so-common-template.json"
+      template => "/so-strelka-template.json"
       template_overwrite => true
     }
   }

salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja

@@ -1,6 +1,5 @@
 {% set MANAGER = salt['pillar.get']('static:managerip', '') %}
 {% set BATCH = salt['pillar.get']('logstash_settings:ls_pipeline_batch_size', 125) %}
-
 output {
 	redis {
 		host => '{{ MANAGER }}'

salt/logstash/pipelines/templates/so/so-flow-template.json.jinja

@@ -0,0 +1,13 @@
+{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-flow:shards', 1) %}
+{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %}
+{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-flow:refresh', '30s') %}
+{
+  "index_patterns": ["so-flow-*"],
+  "version": 50001,
+  "order": 11,
+  "settings":{
+    "number_of_replicas":{{ REPLICAS }},
+    "number_of_shards":{{ SHARDS }},
+    "index.refresh_interval":"{{ REFRESH }}"
+  }
+}

salt/zeek/init.sls

@@ -86,20 +86,20 @@ nodecfgsync:
     - group: 939
     - template: jinja
 
-zeekcleanscript:
+#zeekcleanscript:
-  file.managed:
+#  file.managed:
-    - name: /usr/local/bin/zeek_clean
+#    - name: /usr/local/bin/zeek_clean
-    - source: salt://zeek/cron/zeek_clean
+#    - source: salt://zeek/cron/zeek_clean
-    - mode: 755
+#    - mode: 755
 
-/usr/local/bin/zeek_clean:
+#/usr/local/bin/zeek_clean:
-  cron.present:
+#  cron.present:
-    - user: root
+#    - user: root
-    - minute: '*'
+#    - minute: '*'
-    - hour: '*'
+#    - hour: '*'
-    - daymonth: '*'
+#    - daymonth: '*'
-    - month: '*'
+#    - month: '*'
-    - dayweek: '*'
+#    - dayweek: '*'
 
 plcronscript:
   file.managed:

setup/so-functions

@@ -1048,6 +1048,11 @@ manager_static() {
 		"      warm: 7"\
 		"      close: 30"\
 		"      delete: 365"\
+		"    so-flow:"\
+		"      shards: 1"\
+		"      warm: 7"\
+		"      close: 30"\
+		"      delete: 365"\
 		"    so-ids:"\
 		"      shards: 1"\
 		"      warm: 7"\