Merge pull request #1529 from Security-Onion-Solutions/dev

2.3.0 GA!
2026-04-25 14:07:49 +02:00 · 2020-10-16 10:53:53 -04:00
parent bafb13fd6d e2f16d51a6
commit ba7b34a8ce
130 changed files with 11019 additions and 7494 deletions
@@ -1,37 +1,28 @@
-## Security Onion 2.2.0.rc3
+## Security Onion 2.3.0
-Security Onion 2.2.0 RC3 is here!
+Security Onion 2.3.0 is here!
 ### Warnings and Disclaimers
 - If this breaks your system, you get to keep both pieces!  
 - This is a work in progress and is in constant flux.  
 - This configuration may change drastically over time leading up to the final release.  
 - Do NOT run this on a system that you care about!  
 - Do NOT run this on a system that has data that you care about!  
 - This script should only be run on a TEST box with TEST data!  
 - Use of this script may result in nausea, vomiting, or a burning sensation.  
 ### Release Notes
-https://docs.securityonion.net/en/2.2/release-notes.html
+https://docs.securityonion.net/en/2.3/release-notes.html
 ### Requirements
-https://docs.securityonion.net/en/2.2/hardware.html
+https://docs.securityonion.net/en/2.3/hardware.html
 ### Download
-https://docs.securityonion.net/en/2.2/download.html
+https://docs.securityonion.net/en/2.3/download.html
 ### Installation
-https://docs.securityonion.net/en/2.2/installation.html
+https://docs.securityonion.net/en/2.3/installation.html
 ### FAQ
-https://docs.securityonion.net/en/2.2/faq.html
+https://docs.securityonion.net/en/2.3/faq.html
 ### Feedback
-https://docs.securityonion.net/en/2.2/community-support.html
+https://docs.securityonion.net/en/2.3/community-support.html
@@ -1,16 +1,16 @@
-### 2.2.0-rc3 ISO image built on 2020/09/17
+### 2.3.0 ISO image built on 2020/10/15
 ### Download and Verify
-2.2.0-rc3 ISO image:  
+2.3.0 ISO image:  
-https://download.securityonion.net/file/securityonion/securityonion-2.2.0-rc3.iso
+https://download.securityonion.net/file/securityonion/securityonion-2.3.0.iso
-MD5: 051883501C905653ACBCEC513C294778  
+MD5: E05B220E4FD7C054DF5C50906EE1375B
-SHA1: 0A66F6636F53B268E7FFB743A3136AC5CC3E0E96  
+SHA1: 55E93C6EAB140AB4A0F07873CC871EBFDC699CD6  
-SHA256: 5A9F303954AF1B1D271CE526E5DCBFC28F3FFC0621B291A29F0F7F2E8EB11C43 
+SHA256: 57B96A6E0951143E123BFC0CD0404F7466776E69F3C115F5A0444C0C6D5A6E32 
 Signature for ISO image:  
-https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.2.0-rc3.iso.sig
+https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.0.iso.sig
 Signing key:  
 https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS  
@@ -24,22 +24,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/ma
 Download the signature file for the ISO:  
 ```
-wget https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.2.0-rc3.iso.sig
+wget https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.0.iso.sig
 ```
 Download the ISO image:  
 ```
-wget https://download.securityonion.net/file/securityonion/securityonion-2.2.0-rc3.iso
+wget https://download.securityonion.net/file/securityonion/securityonion-2.3.0.iso
 ```
 Verify the downloaded ISO image using the signature file:  
 ```
-gpg --verify securityonion-2.2.0-rc3.iso.sig securityonion-2.2.0-rc3.iso
+gpg --verify securityonion-2.3.0.iso.sig securityonion-2.3.0.iso
 ```
 The output should show "Good signature" and the Primary key fingerprint should match what's shown below:
 ```
-gpg: Signature made Thu 17 Sep 2020 10:05:27 AM EDT using RSA key ID FE507013
+gpg: Signature made Thu 15 Oct 2020 08:06:28 PM EDT using RSA key ID FE507013
 gpg: Good signature from "Security Onion Solutions, LLC <info@securityonionsolutions.com>"
 gpg: WARNING: This key is not certified with a trusted signature!
 gpg:          There is no indication that the signature belongs to the owner.
@@ -47,4 +47,4 @@ Primary key fingerprint: C804 A93D 36BE 0C73 3EA1  9644 7C10 60B7 FE50 7013
 ```
 Once you've verified the ISO image, you're ready to proceed to our Installation guide:  
-https://docs.securityonion.net/en/2.2/installation.html
+https://docs.securityonion.net/en/2.3/installation.html
@@ -1 +1 @@
-2.2.0-rc.3
+2.3.0
@@ -0,0 +1,14 @@
 [Unit]
 Description=The Salt Master Server
 Documentation=man:salt-master(1) file:///usr/share/doc/salt/html/contents.html https://docs.saltstack.com/en/latest/contents.html
 After=network.target
 [Service]
 LimitNOFILE=100000
 Type=notify
 NotifyAccess=all
 ExecStart=/usr/bin/salt-master
 Restart=always
 [Install]
 WantedBy=multi-user.target
@@ -0,0 +1,13 @@
 elasticsearch:
  templates:
    - so/so-beats-template.json.jinja
    - so/so-common-template.json
    - so/so-firewall-template.json.jinja
    - so/so-flow-template.json.jinja
    - so/so-ids-template.json.jinja
    - so/so-import-template.json.jinja
    - so/so-osquery-template.json.jinja
    - so/so-ossec-template.json.jinja
    - so/so-strelka-template.json.jinja
    - so/so-syslog-template.json.jinja
    - so/so-zeek-template.json.jinja
@@ -0,0 +1,11 @@
 logrotate:
  conf: | 
    daily
    rotate 14
    missingok
    copytruncate
    compress
    create
    extension .log
    dateext
    dateyesterday
@@ -1,6 +1,7 @@
 base:
  '*':
    - patch.needs_restarting
    - logrotate
  '*_eval or *_helix or *_heavynode or *_sensor or *_standalone or *_import':
    - match: compound
@@ -13,22 +14,23 @@ base:
    - logstash.search
    - elasticsearch.search
  '*_sensor':
    - global
    - zeeklogs
    - healthcheck.sensor
    - minions.{{ grains.id }}
  '*_manager or *_managersearch':
    - match: compound
    - global
    - data.*
    - secrets
    - minions.{{ grains.id }}
  '*_manager':
    - logstash
    - logstash.manager
    - elasticsearch.manager
  '*_manager or *_managersearch':
    - match: compound
    - data.*
    - secrets
    - global
    - minions.{{ grains.id }}
  '*_sensor':
    - zeeklogs
    - healthcheck.sensor
    - global
    - minions.{{ grains.id }}
  '*_eval':
    - data.*
@@ -56,29 +58,29 @@ base:
    - minions.{{ grains.id }}
  '*_heavynode':
    - global
    - zeeklogs
    - global
    - minions.{{ grains.id }}
  '*_helix':
    - global
    - fireeye
    - zeeklogs
    - logstash
    - logstash.helix
    - global
    - minions.{{ grains.id }}
  '*_fleet':
    - global
    - data.*
    - secrets
    - global
    - minions.{{ grains.id }}
  '*_searchnode':
    - global
    - logstash
    - logstash.search
    - elasticsearch.search
    - global
    - minions.{{ grains.id }}
  '*_import':
@@ -52,4 +52,5 @@ zeek:
      - frameworks/signatures/detect-windows-shells
    redef:
      - LogAscii::use_json = T;
-      - LogAscii::json_timestamps = JSON::TS_ISO8601;
+      - LogAscii::json_timestamps = JSON::TS_ISO8601;
      - CaptureLoss::watch_interval = 5 mins;
@@ -1,42 +0,0 @@
 zeeklogs:
  enabled:
    - conn
    - dce_rpc
    - dhcp
    - dhcpv6
    - dnp3
    - dns
    - dpd
    - files
    - ftp
    - http
    - intel
    - irc
    - kerberos
    - modbus
    - mqtt
    - notice
    - ntlm
    - openvpn
    - pe
    - radius
    - rfb
    - rdp
    - signatures
    - sip
    - smb_files
    - smb_mapping
    - smtp
    - snmp
    - software
    - ssh
    - ssl
    - syslog
    - telnet
    - tunnel
    - weird
    - mysql
    - socks
    - x509
  disabled:
@@ -1,4 +1,4 @@
 #!py
 def status():
-  return __salt__['cmd.run']('/sbin/so-status')
+  return __salt__['cmd.run']('/usr/sbin/so-status')
@@ -0,0 +1,2 @@
 #!/bin/bash
 logrotate -f /opt/so/conf/log-rotate.conf >/dev/null 2>&1
@@ -0,0 +1 @@
 <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 87.86 105.22"><defs><style>.cls-1{fill:#fff;}.cls-2{fill:#1976d2;}</style></defs><g id="Layer_2" data-name="Layer 2"><g id="Layer_1-2" data-name="Layer 1"><g id="Onion"><path id="Flesh" class="cls-1" d="M43.37,71.34a1.27,1.27,0,0,0,.44-.51,4.74,4.74,0,0,0,.61-2.39c-.12-6.79-.22-12.88-4-14.46-4.05-1.72-9.38,3.14-10.71,4.35a19.84,19.84,0,0,0-6.17,12.34c-.1,1-.76,9.34,5.46,15.41s15.45,6.06,21.72,3.53A22.25,22.25,0,0,0,61.88,79.16c5.31-10,1.61-20.31.85-22.3C57.78,44,43.35,36.11,29.88,36.78c-2.17.11-15.82,1-24.16,12.42A30.55,30.55,0,0,0,0,67.36c.15,16.14,13.38,29.51,26.23,34.7,12.61,5.1,24,2.76,28.78,1.65s17.12-4,25.53-15.08a34.47,34.47,0,0,0,7.24-18.46,34.79,34.79,0,0,0-3.42-17.32c-1.11-2.3-6.16-12.09-17-17C57,31.21,48.52,34.37,45.65,29.12a8.46,8.46,0,0,1-.41-6.21,1,1,0,0,0-1.05-1.28l-1.6,0a1.07,1.07,0,0,0-1,.8c-.66,2.51-1.12,6,.51,9.17C46,39.08,56.87,35.31,67.56,42.78c8.29,5.79,14.14,16.69,13.21,27.29a28.06,28.06,0,0,1-6,14.65c-7,9-17,11.29-21.82,12.38-4,.9-13.19,2.87-23.54-.93-2.65-1-20.33-8.29-22.38-25C5.72,60.55,13,48.9,24.21,44.93c13-4.6,27.26,2.75,32.09,13.26.58,1.25,4.85,10.93-.59,18.72-4.05,5.79-13.07,9.94-19.77,6A13.48,13.48,0,0,1,30,68.25c1.42-5,6.37-8.72,8.13-7.84s2.94,6.14,3,9.85A1.39,1.39,0,0,0,43.37,71.34Z"/><path id="Stem" class="cls-2" d="M30,27.14l-4.17,1.27a1.16,1.16,0,0,1-1.49-.93l-.11-.72a26.93,26.93,0,0,0-4.53-11.09A1.13,1.13,0,0,1,20.06,14l1.06-.63a1.15,1.15,0,0,1,1.52.32c.41.58.82,1.17,1.23,1.78l1.48,2.2C28.42,7.27,37.14.12,46.21,0,58.09-.16,65.59,10.67,68,17.63a23.37,23.37,0,0,1,.94,3.64.91.91,0,0,1-1.14,1l-2.66-.73a1.47,1.47,0,0,1-1-1.08,19.71,19.71,0,0,0-1.9-4.8c-3-5.44-9.67-11.21-16.55-10.59-7.74.7-15.22,9.46-14.85,20.91A1.14,1.14,0,0,1,30,27.14Z"/></g></g></g></svg>
@@ -0,0 +1 @@
 <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 87.86 105.22"><defs><style>.cls-1{fill:#1976d2;}</style></defs><g id="Layer_2" data-name="Layer 2"><g id="Layer_1-2" data-name="Layer 1"><g id="Onion"><path id="Flesh" d="M43.37,71.34a1.27,1.27,0,0,0,.44-.51,4.74,4.74,0,0,0,.61-2.39c-.12-6.79-.22-12.88-4-14.46-4.05-1.72-9.38,3.14-10.71,4.35a19.84,19.84,0,0,0-6.17,12.34c-.1,1-.76,9.34,5.46,15.41s15.45,6.06,21.72,3.53A22.25,22.25,0,0,0,61.88,79.16c5.31-10,1.61-20.31.85-22.3C57.78,44,43.35,36.11,29.88,36.78c-2.17.11-15.82,1-24.16,12.42A30.55,30.55,0,0,0,0,67.36c.15,16.14,13.38,29.51,26.23,34.7,12.61,5.1,24,2.76,28.78,1.65s17.12-4,25.53-15.08a34.47,34.47,0,0,0,7.24-18.46,34.79,34.79,0,0,0-3.42-17.32c-1.11-2.3-6.16-12.09-17-17C57,31.21,48.52,34.37,45.65,29.12a8.46,8.46,0,0,1-.41-6.21,1,1,0,0,0-1.05-1.28l-1.6,0a1.07,1.07,0,0,0-1,.8c-.66,2.51-1.12,6,.51,9.17C46,39.08,56.87,35.31,67.56,42.78c8.29,5.79,14.14,16.69,13.21,27.29a28.06,28.06,0,0,1-6,14.65c-7,9-17,11.29-21.82,12.38-4,.9-13.19,2.87-23.54-.93-2.65-1-20.33-8.29-22.38-25C5.72,60.55,13,48.9,24.21,44.93c13-4.6,27.26,2.75,32.09,13.26.58,1.25,4.85,10.93-.59,18.72-4.05,5.79-13.07,9.94-19.77,6A13.48,13.48,0,0,1,30,68.25c1.42-5,6.37-8.72,8.13-7.84s2.94,6.14,3,9.85A1.39,1.39,0,0,0,43.37,71.34Z"/><path id="Stem" class="cls-1" d="M30,27.14l-4.17,1.27a1.16,1.16,0,0,1-1.49-.93l-.11-.72a26.93,26.93,0,0,0-4.53-11.09A1.13,1.13,0,0,1,20.06,14l1.06-.63a1.15,1.15,0,0,1,1.52.32c.41.58.82,1.17,1.23,1.78l1.48,2.2C28.42,7.27,37.14.12,46.21,0,58.09-.16,65.59,10.67,68,17.63a23.37,23.37,0,0,1,.94,3.64.91.91,0,0,1-1.14,1l-2.66-.73a1.47,1.47,0,0,1-1-1.08,19.71,19.71,0,0,0-1.9-4.8c-3-5.44-9.67-11.21-16.55-10.59-7.74.7-15.22,9.46-14.85,20.91A1.14,1.14,0,0,1,30,27.14Z"/></g></g></g></svg>
@@ -0,0 +1,23 @@
 {%- set logrotate_conf = salt['pillar.get']('logrotate:conf') %}
 /opt/so/log/aptcacher-ng/*.log
 /opt/so/log/idstools/*.log
 /opt/so/log/nginx/*.log
 /opt/so/log/soc/*.log
 /opt/so/log/kratos/*.log
 /opt/so/log/kibana/*.log
 /opt/so/log/influxdb/*.log
 /opt/so/log/elastalert/*.log
 /opt/so/log/soctopus/*.log
 /opt/so/log/curator/*.log
 /opt/so/log/fleet/*.log
 /opt/so/log/suricata/*.log
 /opt/so/log/mysql/*.log
 /opt/so/log/playbook/*.log
 /opt/so/log/logstash/*.log
 /opt/so/log/filebeat/*.log
 /opt/so/log/telegraf/*.log
 /opt/so/log/redis/*.log
 {
    {{ logrotate_conf | indent(width=4) }}
 }
@@ -56,6 +56,12 @@ salttmp:
 # Install epel
 {% if grains['os'] == 'CentOS' %}
 repair_yumdb:
  cmd.run:
    - name: 'mv -f /var/lib/rpm/__db* /tmp && yum clean all'
    - onlyif:
      - 'yum check-update 2>&1 | grep "Error: rpmdb open failed"'
 epel:
  pkg.installed:
    - skip_suggestions: True
@@ -192,6 +198,40 @@ sensorrotateconf:
 {% endif %}
 commonlogrotatescript:
  file.managed:
    - name: /usr/local/bin/common-rotate
    - source: salt://common/cron/common-rotate
    - mode: 755
 commonlogrotateconf:
  file.managed:
    - name: /opt/so/conf/log-rotate.conf
    - source: salt://common/files/log-rotate.conf
    - template: jinja
    - mode: 644
 /usr/local/bin/common-rotate:
  cron.present:
    - user: root
    - minute: '1'
    - hour: '0'
    - daymonth: '*'
    - month: '*'
    - dayweek: '*'
 {% if role in ['eval', 'manager', 'managersearch', 'standalone'] %}
 # Add config backup
 /usr/sbin/so-config-backup > /dev/null 2>&1:
  cron.present:
    - user: root
    - minute: '1'
    - hour: '0'
    - daymonth: '*'
    - month: '*'
    - dayweek: '*'
 {% endif %}
 # Make sure Docker is always running
 docker:
  service.running:
@@ -203,4 +243,4 @@ common_state_not_allowed:
  test.fail_without_changes:
    - name: common_state_not_allowed
-{% endif %}
+{% endif %}
@@ -0,0 +1,23 @@
 #!/bin/bash
 # Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 . /usr/sbin/so-common
 echo ""
 echo "Hosts/Networks that have access to login to the Security Onion Console:"
 so-firewall includedhosts analyst
@@ -0,0 +1,309 @@
 #!/bin/bash
 # Copyright 2014-2020 Security Onion Solutions, LLC
 #    This program is free software: you can redistribute it and/or modify
 #    it under the terms of the GNU General Public License as published by
 #    the Free Software Foundation, either version 3 of the License, or
 #    (at your option) any later version.
 #
 #    This program is distributed in the hope that it will be useful,
 #    but WITHOUT ANY WARRANTY; without even the implied warranty of
 #    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 #    GNU General Public License for more details.
 #
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 if [ "$(id -u)" -ne 0 ]; then
 	echo "This script must be run using sudo!"
 	exit 1
 fi
 INSTALL_LOG=/root/so-analyst-install.log
 exec &> >(tee -a "$INSTALL_LOG")
 log() {
 	msg=$1
 	level=${2:-I}
 	now=$(TZ=GMT date +"%Y-%m-%dT%H:%M:%SZ")
 	echo -e "$now | $level | $msg" >> "$INSTALL_LOG" 2>&1
 }
 error() {
 	log "$1" "E"
 }
 info() {
 	log "$1" "I"
 }
 title() {
 	echo -e "\n-----------------------------\n $1\n-----------------------------\n" >> "$INSTALL_LOG" 2>&1
 }
 logCmd() {
 	cmd=$1
 	info "Executing command: $cmd"
 	$cmd >> "$INSTALL_LOG" 2>&1
 }
 analyze_system() {
 	title "System Characteristics"
 	logCmd "uptime"
 	logCmd "uname -a"
 	logCmd "free -h"
 	logCmd "lscpu"
 	logCmd "df -h"
 	logCmd "ip a"
 }
 analyze_system
 OS=$(grep PRETTY_NAME /etc/os-release | grep 'CentOS Linux 7')
 if [ $? -ne 0 ]; then
  echo "This is an unsupported OS. Please use CentOS 7 to install the analyst node."
  exit 1
 fi
 if [[ "$manufacturer" == "Security Onion Solutions" && "$family" == "Automated" ]]; then
  INSTALL=yes
  CURLCONTINUE=no
 else
  INSTALL=''
  CURLCONTINUE=''
 fi
 FIRSTPASS=yes
 while [[ $INSTALL != "yes" ]] && [[ $INSTALL != "no" ]]; do
  if [[ "$FIRSTPASS" == "yes" ]]; then
    clear
    echo "###########################################"
    echo "##          ** W A R N I N G **          ##"
    echo "##    _______________________________    ##"
    echo "##                                       ##"
    echo "##    Installing the Security Onion      ##"
    echo "##   analyst node on this device will    ##"
    echo "##      make permanenet changes to       ##"
    echo "##              the system.              ##"
    echo "##                                       ##"
    echo "###########################################"
    echo "Do you wish to continue? (Type the entire word 'yes' to proceed or 'no' to exit)"
    FIRSTPASS=no
  else
    echo "Please type 'yes' to continue or 'no' to exit."
  fi      
  read INSTALL
 done
 if [[ $INSTALL == "no" ]]; then
  echo "Exiting analyst node installation."
  exit 0
 fi
 echo "Testing for internet connection with curl https://securityonionsolutions.com/"
 CANCURL=$(curl -sI https://securityonionsolutions.com/ | grep "200 OK")
  if [ $? -ne 0 ]; then
    FIRSTPASS=yes
    while [[ $CURLCONTINUE != "yes" ]] && [[ $CURLCONTINUE != "no" ]]; do
      if [[ "$FIRSTPASS" == "yes" ]]; then
        echo "We could not access https://securityonionsolutions.com/."
        echo "Since packages are downloaded from the internet, internet acceess is required."
        echo "If you would like to ignore this warning and continue anyway, please type 'yes'."
        echo "Otherwise, type 'no' to exit."
        FIRSTPASS=no
      else
        echo "Please type 'yes' to continue or 'no' to exit."
      fi  
      read CURLCONTINUE
    done
    if [[ "$CURLCONTINUE" == "no" ]]; then
      echo "Exiting analyst node installation."
      exit 0
    fi
  else
    echo "We were able to curl https://securityonionsolutions.com/."
    sleep 3
  fi
 # Install a GUI text editor
 yum -y install gedit
 # Install misc utils
 yum -y install wget curl unzip epel-release yum-plugin-versionlock;
 # Install xWindows
 yum -y groupinstall "X Window System";
 yum -y install gnome-classic-session gnome-terminal nautilus-open-terminal control-center liberation-mono-fonts;
 unlink /etc/systemd/system/default.target;
 ln -sf /lib/systemd/system/graphical.target /etc/systemd/system/default.target;
 yum -y install file-roller
 # Install Mono - prereq for NetworkMiner
 yum -y install mono-core mono-basic mono-winforms expect
 # Install NetworkMiner
 yum -y install libcanberra-gtk2;
 wget https://www.netresec.com/?download=NetworkMiner -O /tmp/nm.zip;
 mkdir -p /opt/networkminer/
 unzip /tmp/nm.zip -d /opt/networkminer/;
 rm /tmp/nm.zip;
 mv /opt/networkminer/NetworkMiner_*/* /opt/networkminer/
 chmod +x /opt/networkminer/NetworkMiner.exe;
 chmod -R go+w /opt/networkminer/AssembledFiles/;
 chmod -R go+w /opt/networkminer/Captures/;
 # Create networkminer shim
 cat << EOF >> /bin/networkminer
 #!/bin/bash
 /bin/mono /opt/networkminer/NetworkMiner.exe --noupdatecheck "\$@"
 EOF
 chmod +x /bin/networkminer
 # Convert networkminer ico file to png format
 yum -y install ImageMagick
 convert /opt/networkminer/networkminericon.ico /opt/networkminer/networkminericon.png
 # Create menu entry
 cat << EOF >> /usr/share/applications/networkminer.desktop
 [Desktop Entry]
 Name=NetworkMiner
 Comment=NetworkMiner
 Encoding=UTF-8
 Exec=/bin/networkminer %f
 Icon=/opt/networkminer/networkminericon-4.png
 StartupNotify=true
 Terminal=false
 X-MultipleArgs=false
 Type=Application
 MimeType=application/x-pcap;
 Categories=Network;
 EOF
 # Set default monospace font to Liberation
 cat << EOF >> /etc/fonts/local.conf
 <match target="pattern">
  <test name="family" qual="any">
   <string>monospace</string>
  </test>
  <edit binding="strong" mode="prepend" name="family">
   <string>Liberation Mono</string>
  </edit>
 </match>
 EOF
 # Install Wireshark for Gnome
 yum -y install wireshark-gnome; 
 # Install dnsiff
 yum -y install dsniff;
 # Install hping3
 yum -y install hping3;
 # Install netsed
 yum -y install netsed;
 # Install ngrep
 yum -y install ngrep;
 # Install scapy
 yum -y install python36-scapy;
 # Install ssldump
 yum -y install ssldump;
 # Install tcpdump
 yum -y install tcpdump;
 # Install tcpflow
 yum -y install tcpflow;
 # Install tcpxtract
 yum -y install tcpxtract;
 # Install whois
 yum -y install whois;
 # Install foremost
 yum -y install https://forensics.cert.org/centos/cert/7/x86_64//foremost-1.5.7-13.1.el7.x86_64.rpm;
 # Install chromium
 yum -y install chromium;
 # Install tcpstat
 yum -y install https://github.com/Security-Onion-Solutions/securityonion-docker-rpm/releases/download/securityonion-tcpstat-1.5.0/securityonion-tcpstat-1.5.0.rpm;
 # Install tcptrace
 yum -y install https://github.com/Security-Onion-Solutions/securityonion-docker-rpm/releases/download/securityonion-tcptrace-6.6.7/securityonion-tcptrace-6.6.7.rpm;
 # Install sslsplit
 yum -y install libevent;
 yum -y install sslsplit;
 # Install Bit-Twist
 yum -y install https://github.com/Security-Onion-Solutions/securityonion-docker-rpm/releases/download/securityonion-bittwist-2.0.0/securityonion-bittwist-2.0.0.rpm;
 # Install chaosreader
 yum -y install perl-IO-Compress perl-Net-DNS;
 yum -y install https://github.com/Security-Onion-Solutions/securityonion-docker-rpm/releases/download/securityonion-chaosreader-0.95.10/securityonion-chaosreader-0.95.10.rpm;
 chmod +x /bin/chaosreader;
 if [ -f ../../files/analyst/README ]; then
  cp ../../files/analyst/README /;
  cp ../../files/analyst/so-wallpaper.jpg /usr/share/backgrounds/;
  cp ../../files/analyst/so-lockscreen.jpg /usr/share/backgrounds/;
  cp ../../files/analyst/so-login-logo-dark.svg /usr/share/pixmaps/;
 else
  cp /opt/so/saltstack/default/salt/common/files/analyst/README /;
  cp /opt/so/saltstack/default/salt/common/files/analyst/so-wallpaper.jpg /usr/share/backgrounds/;
  cp /opt/so/saltstack/default/salt/common/files/analyst/so-lockscreen.jpg /usr/share/backgrounds/;
  cp /opt/so/saltstack/default/salt/common/files/analyst/so-login-logo-dark.svg /usr/share/pixmaps/;
 fi
 # Set background wallpaper
 cat << EOF >> /etc/dconf/db/local.d/00-background
 # Specify the dconf path
 [org/gnome/desktop/background]
 # Specify the path to the desktop background image file
 picture-uri='file:///usr/share/backgrounds/so-wallpaper.jpg'
 # Specify one of the rendering options for the background image:
 # 'none', 'wallpaper', 'centered', 'scaled', 'stretched', 'zoom', 'spanned'
 picture-options='zoom'
 # Specify the left or top color when drawing gradients or the solid color
 primary-color='000000'
 # Specify the right or bottom color when drawing gradients
 secondary-color='FFFFFF'
 EOF
 # Set lock screen
 cat << EOF >> /etc/dconf/db/local.d/00-screensaver
 [org/gnome/desktop/session]
 idle-delay=uint32 180
 [org/gnome/desktop/screensaver]
 lock-enabled=true
 lock-delay=uint32 120
 picture-options='zoom'
 picture-uri='file:///usr/share/backgrounds/so-lockscreen.jpg'
 EOF
 cat << EOF >> /etc/dconf/db/local.d/locks/screensaver
 /org/gnome/desktop/session/idle-delay
 /org/gnome/desktop/screensaver/lock-enabled
 /org/gnome/desktop/screensaver/lock-delay
 EOF
 # Do not show the user list at login screen
 cat << EOF >> /etc/dconf/db/local.d/00-login-screen
 [org/gnome/login-screen]
 logo='/usr/share/pixmaps/so-login-logo-dark.svg'
 disable-user-list=true
 EOF
 dconf update;
 echo
 echo "Analyst workstation has been installed!"
 echo "Press ENTER to reboot or Ctrl-C to cancel."
 read pause
 reboot;
@@ -33,15 +33,21 @@ header() {
 lookup_pillar() {
    key=$1
-    cat /opt/so/saltstack/local/pillar/global.sls | grep $key | awk '{print $2}'
+    salt-call --no-color  pillar.get global:${key} --out=newline_values_only
 }
 lookup_pillar_secret() {
    key=$1
-    cat /opt/so/saltstack/local/pillar/secrets.sls | grep $key | awk '{print $2}'
+    salt-call --no-color  pillar.get secrets:${key} --out=newline_values_only
 }
 check_container() {
 	docker ps | grep "$1:" > /dev/null 2>&1
 	return $?
 }
 check_password() {
    local password=$1
    echo "$password" | egrep -v "'|\"|\\\\" > /dev/null 2>&1
    return $?
 }
@@ -0,0 +1,44 @@
 #!/bin/bash
 #
 # Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.. /usr/sbin/so-common
 {% set BACKUPLOCATIONS = salt['pillar.get']('backup:locations', {}) %}
 TODAY=$(date '+%Y_%m_%d')
 BACKUPFILE="/nsm/backup/so-config-backup-$TODAY.tar"
 MAXBACKUPS=7
 # Create backup dir if it does not exist
 mkdir -p /nsm/backup
 # If we haven't already written a backup file for today, let's do so
 if [ ! -f $BACKUPFILE ]; then
  # Create empty backup file
  tar -cf $BACKUPFILE -T /dev/null
  # Loop through all paths defined in global.sls, and append them to backup file
  {%- for LOCATION in BACKUPLOCATIONS %}
  tar -rf $BACKUPFILE {{ LOCATION }}
  {%- endfor %}
 fi
 # Find oldest backup file and remove it
 NUMBACKUPS=$(find /nsm/backup/ -type f -name "so-config-backup*" | wc -l)
 OLDESTBACKUP=$(find /nsm/backup/ -type f -name "so-config-backup*" | ls -1t | tail -1)
 if [ "$NUMBACKUPS" -gt "$MAXBACKUPS" ]; then
  rm -f /nsm/backup/$OLDESTBACKUP
 fi
@@ -40,7 +40,7 @@ test -t 0
 if [[ $? == 0 ]]; then
  echo "Enter new password:"
 fi
-read -s CORTEX_PASS
+read -rs CORTEX_PASS
 # Create new user in Cortex
 resp=$(curl -sk -XPOST -H "Authorization: Bearer $CORTEX_KEY" -H "Content-Type: application/json" "https://$CORTEX_IP/cortex/api/user" -d "{\"name\": \"$CORTEX_USER\",\"roles\": [\"read\",\"analyze\",\"orgadmin\"],\"organization\": \"$CORTEX_ORG_NAME\",\"login\": \"$CORTEX_USER\",\"password\" : \"$CORTEX_PASS\" }")
@@ -14,7 +14,7 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
-{%- set MANAGERIP = salt['pillar.get']('global:managerip', '') -%}
+{%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%}
 . /usr/sbin/so-common
 SKIP=0
@@ -50,7 +50,11 @@ done
 if [ $SKIP -ne 1 ]; then
        # List indices
        echo 
-        curl {{ MANAGERIP }}:9200/_cat/indices?v
+        {% if grains['role'] in ['so-node','so-heavynode'] %}
        curl -k https://{{ NODEIP }}:9200/_cat/indices?v
        {% else %}
        curl {{ NODEIP }}:9200/_cat/indices?v
        {% endif %}
        echo
        # Inform user we are about to delete all data
        echo
@@ -89,10 +93,18 @@ fi
 # Delete data
 echo "Deleting data..."
-INDXS=$(curl -s -XGET {{ MANAGERIP }}:9200/_cat/indices?v | egrep 'logstash|elastalert|so-' | awk '{ print $3 }')
+{% if grains['role'] in ['so-node','so-heavynode'] %}
 INDXS=$(curl -s -XGET -k https://{{ NODEIP }}:9200/_cat/indices?v | egrep 'logstash|elastalert|so-' | awk '{ print $3 }')
 {% else %}
 INDXS=$(curl -s -XGET {{ NODEIP }}:9200/_cat/indices?v | egrep 'logstash|elastalert|so-' | awk '{ print $3 }')
 {% endif %}
 for INDX in ${INDXS}
 do
-    curl -XDELETE "{{ MANAGERIP }}:9200/${INDX}" > /dev/null 2>&1
+    {% if grains['role'] in ['so-node','so-heavynode'] %}
    curl -XDELETE -k https://"{{ NODEIP }}:9200/${INDX}" > /dev/null 2>&1
    {% else %}
    curl -XDELETE "{{ NODEIP }}:9200/${INDX}" > /dev/null 2>&1
    {% endif %}
 done
 #Start Logstash/Filebeat
@@ -0,0 +1,33 @@
 #!/bin/bash
 #
 # Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>
 {%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%}
 . /usr/sbin/so-common
 if [ "$1" == "" ]; then
        {% if grains['role'] in ['so-node','so-heavynode'] %}
        curl -s -k https://{{ NODEIP }}:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines"
        {% else %}
        curl -s {{ NODEIP }}:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines"
        {% endif %}
 else
        {% if grains['role'] in ['so-node','so-heavynode'] %}
        curl -s -k https://{{ NODEIP }}:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines.\"$1\""
        {% else %}
        curl -s {{ NODEIP }}:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines.\"$1\""
        {% endif %}
 fi
@@ -0,0 +1,31 @@
 #!/bin/bash
 #
 # Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 {%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%}
 . /usr/sbin/so-common
 if [ "$1" == "" ]; then
    {% if grains['role'] in ['so-node','so-heavynode'] %}
    curl -s -k https://{{ NODEIP }}:9200/_ingest/pipeline/* | jq 'keys'
    {% else %}
    curl -s {{ NODEIP }}:9200/_ingest/pipeline/* | jq 'keys'
    {% endif %}
 else
    {% if grains['role'] in ['so-node','so-heavynode'] %}
    curl -s -k https://{{ NODEIP }}:9200/_ingest/pipeline/$1 | jq
    {% else %}
    curl -s {{ NODEIP }}:9200/_ingest/pipeline/$1 | jq
    {% endif %}
 fi
@@ -0,0 +1,31 @@
 #!/bin/bash
 #
 # Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 {%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%}
 . /usr/sbin/so-common
 if [ "$1" == "" ]; then
    {% if grains['role'] in ['so-node','so-heavynode'] %}
    curl -s -k https://{{ NODEIP }}:9200/_template/* | jq 'keys'
    {% else %}
    curl -s {{ NODEIP }}:9200/_template/* | jq 'keys'
    {% endif %}
 else
    {% if grains['role'] in ['so-node','so-heavynode'] %}
    curl -s -k https://{{ NODEIP }}:9200/_template/$1 | jq
    {% else %}
    curl -s {{ NODEIP }}:9200/_template/$1 | jq
    {% endif %}
 fi
@@ -30,7 +30,11 @@ echo -n "Waiting for ElasticSearch..."
 COUNT=0
 ELASTICSEARCH_CONNECTED="no"
 while [[ "$COUNT" -le 240 ]]; do
    {% if grains['role'] in ['so-node','so-heavynode'] %}
    curl -k --output /dev/null --silent --head --fail https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"
    {% else %}
 	curl --output /dev/null --silent --head --fail http://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"
 	{% endif %}
 	if [ $? -eq 0 ]; then
 		ELASTICSEARCH_CONNECTED="yes"
 		echo "connected!"
@@ -51,7 +55,11 @@ cd ${ELASTICSEARCH_TEMPLATES}
 echo "Loading templates..."
 {% if grains['role'] in ['so-node','so-heavynode'] %}
 for i in *; do TEMPLATE=$(echo $i | cut -d '-' -f2); echo "so-$TEMPLATE"; curl -k ${ELASTICSEARCH_AUTH} -s -XPUT https://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_template/so-$TEMPLATE -H 'Content-Type: application/json' -d@$i 2>/dev/null; echo; done
 {% else %}
 for i in *; do TEMPLATE=$(echo $i | cut -d '-' -f2); echo "so-$TEMPLATE"; curl ${ELASTICSEARCH_AUTH} -s -XPUT http://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_template/so-$TEMPLATE -H 'Content-Type: application/json' -d@$i 2>/dev/null; echo; done
 {% endif %}
 echo
 cd - >/dev/null
@@ -39,11 +39,16 @@ test -t 0
 if [[ $? == 0 ]]; then
  echo "Enter new password:"
 fi
-read -s FLEET_PASS
+read -rs FLEET_PASS
 if ! check_password "$FLEET_PASS"; then
  echo "Password is invalid. Please exclude single quotes, double quotes and backslashes from the password."
  exit 2
 fi
 FLEET_HASH=$(docker exec so-soctopus python -c "import bcrypt; print(bcrypt.hashpw('$FLEET_PASS'.encode('utf-8'), bcrypt.gensalt()).decode('utf-8'));" 2>&1)
 if [[ $? -ne 0 ]]; then
-	echo "Failed to generate Fleet password hash."
+	echo "Failed to generate Fleet password hash"
 	exit 2
 fi
@@ -51,9 +56,9 @@ MYSQL_OUTPUT=$(docker exec so-mysql mysql -u root --password=$MYSQL_PASS fleet -
    "INSERT INTO users (password,salt,username,email,admin,enabled) VALUES ('$FLEET_HASH','','$FLEET_USER','$FLEET_USER',1,1)" 2>&1)
 if [[ $? -eq 0 ]]; then
-    echo "Successfully added user to Fleet."
+    echo "Successfully added user to Fleet"
 else
-    echo "Unable to add user to Fleet; user might already exist."
+    echo "Unable to add user to Fleet; user might already exist"
    echo $resp
    exit 2
 fi
@@ -20,7 +20,7 @@
 usage() {
    echo "Usage: $0 <user-name>"
    echo ""
-    echo "Enables or disables a user in Fleet."
+    echo "Enables or disables a user in Fleet"
    exit 1
 }
@@ -50,9 +50,9 @@ MYSQL_OUTPUT=$(docker exec so-mysql mysql -u root --password=$MYSQL_PASS fleet -
    "UPDATE users SET enabled=$FLEET_STATUS WHERE username='$FLEET_USER'" 2>&1)
 if [[ $? -eq 0 ]]; then
-    echo "Successfully updated user in Fleet."
+    echo "Successfully updated user in Fleet"
 else
-    echo "Failed to update user in Fleet."
+    echo "Failed to update user in Fleet"
    echo $resp
    exit 2
 fi
@@ -15,4 +15,8 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
-curl -X GET "localhost:9200/_cat/indices?v"
+{% if grains['role'] in ['so-node','so-heavynode'] %}
 curl -X GET -k https://localhost:9200/_cat/indices?v
 {% else %}
 curl -X GET localhost:9200/_cat/indices?v
 {% endif %}
@@ -10,4 +10,4 @@ got_root() {
 }
 got_root
-docker exec -d so-idstools /bin/bash -c 'cd /opt/so/idstools/etc && idstools-rulecat'
+docker exec so-idstools /bin/bash -c 'cd /opt/so/idstools/etc && idstools-rulecat'
@@ -23,97 +23,104 @@ CUR_USAGE=$(df -P $SENSOR_DIR | tail -1 | awk '{print $5}' | tr -d %)
 LOG="/opt/so/log/sensor_clean.log"
 TODAY=$(date -u "+%Y-%m-%d")
-clean () {
+clean() {
-                ## find the oldest Zeek logs directory
+	## find the oldest Zeek logs directory
-                OLDEST_DIR=$(ls /nsm/zeek/logs/ | grep -v "current" | grep -v "stats" | grep -v "packetloss" | grep -v "zeek_clean" | sort | head -n 1)
+	OLDEST_DIR=$(ls /nsm/zeek/logs/ | grep -v "current" | grep -v "stats" | grep -v "packetloss" | grep -v "zeek_clean" | sort | head -n 1)
-                if [ -z "$OLDEST_DIR" -o "$OLDEST_DIR" == ".." -o "$OLDEST_DIR" == "." ]
+	if [ -z "$OLDEST_DIR" -o "$OLDEST_DIR" == ".." -o "$OLDEST_DIR" == "." ]; then
-                then
+		echo "$(date) - No old Zeek logs available to clean up in /nsm/zeek/logs/" >>$LOG
-                        echo "$(date) - No old Zeek logs available to clean up in /nsm/zeek/logs/" >> $LOG
+		#exit 0
-                        #exit 0
+	else
-                else
+		echo "$(date) - Removing directory: /nsm/zeek/logs/$OLDEST_DIR" >>$LOG
-                        echo "$(date) - Removing directory: /nsm/zeek/logs/$OLDEST_DIR" >> $LOG
+		rm -rf /nsm/zeek/logs/"$OLDEST_DIR"
-                        rm -rf /nsm/zeek/logs/"$OLDEST_DIR"
+	fi
                fi
 	## Remarking for now, as we are moving extracted files to /nsm/strelka/processed
 	## find oldest files in extracted directory and exclude today
 	#OLDEST_EXTRACT=$(find /nsm/zeek/extracted/complete -type f -printf '%T+ %p\n' 2>/dev/null | sort | grep -v $TODAY | head -n 1)
 	#if [ -z "$OLDEST_EXTRACT" -o "$OLDEST_EXTRACT" == ".." -o "$OLDEST_EXTRACT" == "." ]
 	#then
 	#        echo "$(date) - No old extracted files available to clean up in /nsm/zeek/extracted/complete" >> $LOG
 	#else
 	#        OLDEST_EXTRACT_DATE=`echo $OLDEST_EXTRACT | awk '{print $1}' | cut -d+ -f1`
 	#        OLDEST_EXTRACT_FILE=`echo $OLDEST_EXTRACT | awk '{print $2}'`
 	#        echo "$(date) - Removing extracted files for $OLDEST_EXTRACT_DATE" >> $LOG
 	#        find /nsm/zeek/extracted/complete -type f -printf '%T+ %p\n' | grep $OLDEST_EXTRACT_DATE | awk '{print $2}' |while read FILE
 	#        do
 	#                echo "$(date) - Removing extracted file: $FILE" >> $LOG
 	#                rm -f "$FILE"
 	#        done
 	#fi
-                ## Remarking for now, as we are moving extracted files to /nsm/strelka/processed
+	## Clean up Zeek extracted files processed by Strelka
-                ## find oldest files in extracted directory and exclude today
+	STRELKA_FILES='/nsm/strelka/processed'
-                #OLDEST_EXTRACT=$(find /nsm/zeek/extracted/complete -type f -printf '%T+ %p\n' 2>/dev/null | sort | grep -v $TODAY | head -n 1)
+	OLDEST_STRELKA=$(find $STRELKA_FILES -type f -printf '%T+ %p\n' | sort -n | head -n 1)
-                #if [ -z "$OLDEST_EXTRACT" -o "$OLDEST_EXTRACT" == ".." -o "$OLDEST_EXTRACT" == "." ]
+	if [ -z "$OLDEST_STRELKA" -o "$OLDEST_STRELKA" == ".." -o "$OLDEST_STRELKA" == "." ]; then
-                #then
+		echo "$(date) - No old files available to clean up in $STRELKA_FILES" >>$LOG
-                #        echo "$(date) - No old extracted files available to clean up in /nsm/zeek/extracted/complete" >> $LOG
+	else
-                #else
+		OLDEST_STRELKA_DATE=$(echo $OLDEST_STRELKA | awk '{print $1}' | cut -d+ -f1)
-                #        OLDEST_EXTRACT_DATE=`echo $OLDEST_EXTRACT | awk '{print $1}' | cut -d+ -f1`
+		OLDEST_STRELKA_FILE=$(echo $OLDEST_STRELKA | awk '{print $2}')
-                #        OLDEST_EXTRACT_FILE=`echo $OLDEST_EXTRACT | awk '{print $2}'`
+		echo "$(date) - Removing extracted files for $OLDEST_STRELKA_DATE" >>$LOG
-                #        echo "$(date) - Removing extracted files for $OLDEST_EXTRACT_DATE" >> $LOG
+		find $STRELKA_FILES -type f -printf '%T+ %p\n' | grep $OLDEST_STRELKA_DATE | awk '{print $2}' | while read FILE; do
-                #        find /nsm/zeek/extracted/complete -type f -printf '%T+ %p\n' | grep $OLDEST_EXTRACT_DATE | awk '{print $2}' |while read FILE
+			echo "$(date) - Removing file: $FILE" >>$LOG
-                #        do
+			rm -f "$FILE"
-                #                echo "$(date) - Removing extracted file: $FILE" >> $LOG
+		done
-                #                rm -f "$FILE"
+	fi
                #        done
                #fi
                ## Clean up Zeek extracted files processed by Strelka
                STRELKA_FILES='/nsm/strelka/processed'
                OLDEST_STRELKA=$(find $STRELKA_FILES -type f -printf '%T+ %p\n' | sort -n | head -n 1 )
                if [ -z "$OLDEST_STRELKA" -o "$OLDEST_STRELKA" == ".." -o "$OLDEST_STRELKA" == "." ]
                then
                        echo "$(date) - No old files available to clean up in $STRELKA_FILES" >> $LOG
                else
                        OLDEST_STRELKA_DATE=`echo $OLDEST_STRELKA | awk '{print $1}' | cut -d+ -f1`
                        OLDEST_STRELKA_FILE=`echo $OLDEST_STRELKA | awk '{print $2}'`
                        echo "$(date) - Removing extracted files for $OLDEST_STRELKA_DATE" >> $LOG
                        find $STRELKA_FILES -type f -printf '%T+ %p\n' | grep $OLDEST_STRELKA_DATE | awk '{print $2}' |while read FILE
                        do
                                echo "$(date) - Removing file: $FILE" >> $LOG
                                rm -f "$FILE"
                        done
                fi
-                ## Clean up Suricata log files
+	## Clean up Suricata log files
-                SURICATA_LOGS='/nsm/suricata'
+	SURICATA_LOGS='/nsm/suricata'
-                OLDEST_SURICATA=$(find $STRELKA_FILES -type f -printf '%T+ %p\n' | sort -n | head -n 1)
+	OLDEST_SURICATA=$(find $SURICATA_LOGS -type f -printf '%T+ %p\n' | sort -n | head -n 1)
-                if [ -z "$OLDEST_SURICATA" -o "$OLDEST_SURICATA" == ".." -o "$OLDEST_SURICATA" == "." ]
+	if [[ -z "$OLDEST_SURICATA" ]] || [[ "$OLDEST_SURICATA" == ".." ]] || [[ "$OLDEST_SURICATA" == "." ]]; then
-                then
+		echo "$(date) - No old files available to clean up in $SURICATA_LOGS" >>$LOG
-                        echo "$(date) - No old files available to clean up in $SURICATA_LOGS" >> $LOG
+	else
-                else
+		OLDEST_SURICATA_DATE=$(echo $OLDEST_SURICATA | awk '{print $1}' | cut -d+ -f1)
-                        OLDEST_SURICATA_DATE=`echo $OLDEST_SURICATA | awk '{print $1}' | cut -d+ -f1`
+		OLDEST_SURICATA_FILE=$(echo $OLDEST_SURICATA | awk '{print $2}')
-                        OLDEST_SURICATA_FILE=`echo $OLDEST_SURICATA | awk '{print $2}'`
+		echo "$(date) - Removing logs for $OLDEST_SURICATA_DATE" >>$LOG
-                        echo "$(date) - Removing logs for $OLDEST_SURICATA_DATE" >> $LOG
+		find $SURICATA_LOGS -type f -printf '%T+ %p\n' | grep $OLDEST_SURICATA_DATE | awk '{print $2}' | while read FILE; do
-                        find $SURICATA_LOGS -type f -printf '%T+ %p\n' | grep $OLDEST_SURICATA_DATE | awk '{print $2}' |while read FILE
+			echo "$(date) - Removing file: $FILE" >>$LOG
-                        do
+			rm -f "$FILE"
-                                echo "$(date) - Removing file: $FILE" >> $LOG
+		done
-                                rm -f "$FILE"
+	fi
                        done
                fi
-                ## Clean up extracted pcaps from Steno
+	# Clean Wazuh archives
-                PCAPS='/nsm/pcapout'
+	# Slightly different code since we have 2 files to remove (.json and .log)
-                OLDEST_PCAP=$(find $PCAPS -type f -printf '%T+ %p\n' | sort -n | head -n 1 )
+	WAZUH_ARCHIVE='/nsm/wazuh/logs/archives'
-                if [ -z "$OLDEST_PCAP" -o "$OLDEST_PCAP" == ".." -o "$OLDEST_PCAP" == "." ]
+	OLDEST_WAZUH=$(find $WAZUH_ARCHIVE -type f ! -name "archives.json" -printf "%T+\t%p\n" | sort -n | awk '{print $1}' | head -n 1)
-                then
+	# Make sure we don't delete the current files
-                        echo "$(date) - No old files available to clean up in $PCAPS" >> $LOG
+	find $WAZUH_ARCHIVE -type f ! -name "archives.json" -printf "%T+\t%p\n" | sort -n | awk '{print $2}' | head -n 1 >/tmp/files$$
-                else
+	if [[ $(wc -l </tmp/files$$) -ge 1 ]]; then
-                        OLDEST_PCAP_DATE=`echo $OLDEST_PCAP | awk '{print $1}' | cut -d+ -f1`
+		echo "$(date) - Removing logs for $OLDEST_WAZUH" >>$LOG
-                        OLDEST_PCAP_FILE=`echo $OLDEST_PCAP | awk '{print $2}'`
+		while read -r line; do
-                        echo "$(date) - Removing extracted files for $OLDEST_PCAP_DATE" >> $LOG
+			echo "$(date) - Removing file: $line" >>$LOG
-                        find $PCAPS -type f -printf '%T+ %p\n' | grep $OLDEST_PCAP_DATE | awk '{print $2}' |while read FILE
+			rm "$line"
-                        do
+		done </tmp/files$$
-                                echo "$(date) - Removing file: $FILE" >> $LOG
+	else
-                                rm -f "$FILE"
+		echo "$(date) - No old files available to clean up in $WAZUH_ARCHIVE" >>$LOG
-                        done
+	fi
-                fi
+	rm /tmp/files$$
 	## Clean up extracted pcaps from Steno
 	PCAPS='/nsm/pcapout'
 	OLDEST_PCAP=$(find $PCAPS -type f -printf '%T+ %p\n' | sort -n | head -n 1)
 	if [ -z "$OLDEST_PCAP" -o "$OLDEST_PCAP" == ".." -o "$OLDEST_PCAP" == "." ]; then
 		echo "$(date) - No old files available to clean up in $PCAPS" >>$LOG
 	else
 		OLDEST_PCAP_DATE=$(echo $OLDEST_PCAP | awk '{print $1}' | cut -d+ -f1)
 		OLDEST_PCAP_FILE=$(echo $OLDEST_PCAP | awk '{print $2}')
 		echo "$(date) - Removing extracted files for $OLDEST_PCAP_DATE" >>$LOG
 		find $PCAPS -type f -printf '%T+ %p\n' | grep $OLDEST_PCAP_DATE | awk '{print $2}' | while read FILE; do
 			echo "$(date) - Removing file: $FILE" >>$LOG
 			rm -f "$FILE"
 		done
 	fi
 }
 # Check to see if we are already running
 IS_RUNNING=$(ps aux | grep "sensor_clean" | grep -v grep | wc -l)
-[ "$IS_RUNNING" -gt 2 ] && echo "$(date) - $IS_RUNNING sensor clean script processes running...exiting." >> $LOG && exit 0
+[ "$IS_RUNNING" -gt 2 ] && echo "$(date) - $IS_RUNNING sensor clean script processes running...exiting." >>$LOG && exit 0
 if [ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ]; then
-    while [ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ];
+	while [ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ]; do
-        do
+		clean
-         clean
+		CUR_USAGE=$(df -P $SENSOR_DIR | tail -1 | awk '{print $5}' | tr -d %)
-         CUR_USAGE=$(df -P $SENSOR_DIR | tail -1 | awk '{print $5}' | tr -d %)
+	done
    done
 fi
@@ -23,6 +23,8 @@ REPLAY_ENABLED=$(docker images | grep so-tcpreplay)
 REPLAY_RUNNING=$(docker ps | grep so-tcpreplay)
 if  [ "$REPLAY_ENABLED" != "" ] && [ "$REPLAY_RUNNING" != "" ]; then
   echo
   echo "Preparing to replay PCAPs..."
   docker cp so-tcpreplay:/opt/samples /opt/samples
   docker exec -it so-tcpreplay /usr/local/bin/tcpreplay -i bond0 -M10 /opt/samples/*
   echo
@@ -39,14 +39,19 @@ test -t 0
 if [[ $? == 0 ]]; then
  echo "Enter new password:"
 fi
-read -s THEHIVE_PASS
+read -rs THEHIVE_PASS
 if ! check_password "$THEHIVE_PASS"; then
  echo "Password is invalid. Please exclude single quotes, double quotes and backslashes from the password."
  exit 2
 fi
 # Create new user in TheHive
 resp=$(curl -sk -XPOST -H "Authorization: Bearer $THEHIVE_KEY" -H "Content-Type: application/json" "https://$THEHIVE_IP/thehive/api/user" -d "{\"login\" : \"$THEHIVE_USER\",\"name\" : \"$THEHIVE_USER\",\"roles\" : [\"read\",\"alert\",\"write\",\"admin\"],\"preferences\" : \"{}\",\"password\" : \"$THEHIVE_PASS\"}")
 if [[ "$resp" =~ \"status\":\"Ok\" ]]; then
-    echo "Successfully added user to TheHive."
+    echo "Successfully added user to TheHive"
 else
-    echo "Unable to add user to TheHive; user might already exist."
+    echo "Unable to add user to TheHive; user might already exist"
    echo $resp
    exit 2
 fi
@@ -20,7 +20,7 @@
 usage() {
    echo "Usage: $0 <user-name> <true|false>"
    echo ""
-    echo "Enables or disables a user in thehive."
+    echo "Enables or disables a user in TheHive."
    exit 1
 }
@@ -48,9 +48,9 @@ esac
 resp=$(curl -sk -XPATCH -H "Authorization: Bearer $THEHIVE_KEY" -H "Content-Type: application/json" "https://$THEHIVE_IP/thehive/api/user/${THEHIVE_USER}" -d "{\"status\":\"${THEHIVE_STATUS}\" }")
 if [[ "$resp" =~ \"status\":\"Locked\" || "$resp" =~ \"status\":\"Ok\" ]]; then
-    echo "Successfully updated user in thehive."
+    echo "Successfully updated user in TheHive"
 else
-    echo "Failed to update user in thehive."
+    echo "Failed to update user in TheHive"
    echo "$resp"
    exit 2
 fi
@@ -10,7 +10,7 @@
 . /usr/sbin/so-common
-if [[ $# < 1 || $# > 2 ]]; then
+if [[ $# -lt 1 || $# -gt 2 ]]; then
  echo "Usage: $0 <list|add|update|enable|disable|validate|valemail|valpass> [email]"
  echo ""
  echo "     list: Lists all user email addresses currently defined in the identity system"
@@ -22,7 +22,7 @@ if [[ $# < 1 || $# > 2 ]]; then
  echo " valemail: Validates that the given email address is acceptable for defining a new user; requires 'email' parameter"
  echo "  valpass: Validates that a password is acceptable for defining a new user"
  echo ""
-  echo " Note that the password can be piped into stdin to avoid prompting for it."
+  echo " Note that the password can be piped into STDIN to avoid prompting for it"
  exit 1
 fi
@@ -90,14 +90,16 @@ function validateEmail() {
 function updatePassword() {
  identityId=$1
-  # Read password from stdin (show prompt only if no stdin was piped in)
+  if [ -z "$password" ]; then
-  test -t 0
+    # Read password from stdin (show prompt only if no stdin was piped in)
-  if [[ $? == 0 ]]; then
+    test -t 0
-    echo "Enter new password:"
+    if [[ $? == 0 ]]; then
-  fi
+      echo "Enter new password:"
-  read -s password
+    fi
    read -rs password
-  validatePassword "$password"
+    validatePassword "$password"
  fi
  if [[ -n $identityId ]]; then
    # Generate password hash
@@ -199,6 +201,7 @@ case "${operation}" in
    [[ "$email" == "" ]] && fail "Email address must be provided"
    validateEmail "$email"
    updatePassword
    createUser "$email"
    echo "Successfully added new user to SOC"
    check_container thehive && echo $password | so-thehive-user-add "$email"
@@ -0,0 +1,2 @@
 #!/bin/bash
 so-user list
@@ -2,17 +2,14 @@
 local_salt_dir=/opt/so/saltstack/local
 zeek_logs_enabled() {
  echo "zeeklogs:" > $local_salt_dir/pillar/zeeklogs.sls
  echo "  enabled:" >> $local_salt_dir/pillar/zeeklogs.sls
-  for BLOG in ${BLOGS[@]}; do
+  for BLOG in "${BLOGS[@]}"; do
    echo "    - $BLOG" | tr -d '"' >> $local_salt_dir/pillar/zeeklogs.sls
  done
 }
 whiptail_manager_adv_service_zeeklogs() {
  BLOGS=$(whiptail --title "Security Onion Setup" --checklist "Please Select Logs to Send:" 24 78 12 \
  "conn" "Connection Logging" ON \
  "dce_rpc" "RPC Logs" ON \
@@ -52,7 +49,25 @@ whiptail_manager_adv_service_zeeklogs() {
  "mysql" "MySQL Logs" ON \
  "socks" "SOCKS Logs" ON \
  "x509" "x.509 Logs" ON 3>&1 1>&2 2>&3 )
  local exitstatus=$?
  IFS=' ' read -ra BLOGS <<< "$BLOGS"
  return $exitstatus
 }
 whiptail_manager_adv_service_zeeklogs
-zeek_logs_enabled
+return_code=$?
 case $return_code in
  1)
    whiptail --title "Security Onion Setup" --msgbox "Cancelling. No changes have been made." 8 75
    ;;
  255)
    whiptail --title "Security Onion Setup" --msgbox "Whiptail error occured, exiting." 8 75
    ;;
  *)
    zeek_logs_enabled
    ;;
 esac
@@ -36,10 +36,67 @@ manager_check() {
  fi
 }
 airgap_mounted() {
  # Let's see if the ISO is already mounted. 
  if [ -f /tmp/soagupdate/SecurityOnion/VERSION ]; then
    echo "The ISO is already mounted"
  else
    echo ""
    echo "Looks like we need access to the upgrade content"
    echo "" 
    echo "If you just copied the .iso file over you can specify the path."
    echo "If you burned the ISO to a disk the standard way you can specify the device."
    echo "Example: /home/user/securityonion-2.X.0.iso"
    echo "Example: /dev/cdrom"
    echo ""
    read -p 'Enter the location of the iso: ' ISOLOC
    if [ -f $ISOLOC ]; then
      # Mounting the ISO image
      mkdir -p /tmp/soagupdate
      mount -t iso9660 -o loop $ISOLOC /tmp/soagupdate
      # Make sure mounting was successful
      if [ ! -f /tmp/soagupdate/SecurityOnion/VERSION ]; then
        echo "Something went wrong trying to mount the ISO."
        echo "Ensure you verify the ISO that you downloaded."
        exit 0
      else
        echo "ISO has been mounted!"
      fi  
    elif [ -f $ISOLOC/SecurityOnion/VERSION ]; then
      ln -s $ISOLOC /tmp/soagupdate
      echo "Found the update content"
    else 
      mkdir -p /tmp/soagupdate
      mount $ISOLOC /tmp/soagupdate
      if [ ! -f /tmp/soagupdate/SecurityOnion/VERSION ]; then
        echo "Something went wrong trying to mount the device."
        echo "Ensure you verify the ISO that you downloaded."
        exit 0
      else
        echo "Device has been mounted!"
      fi        
    fi
  fi
 }
 check_airgap() {
  # See if this is an airgap install
  AIRGAP=$(cat /opt/so/saltstack/local/pillar/global.sls | grep airgap | awk '{print $2}')
  if [[ "$AIRGAP" == "True" ]]; then
      is_airgap=0
      UPDATE_DIR=/tmp/soagupdate/SecurityOnion
      AGDOCKER=/tmp/soagupdate/docker
      AGREPO=/tmp/soagupdate/Packages
  else 
      is_airgap=1
  fi
 }
 clean_dockers() {
  # Place Holder for cleaning up old docker images
  echo "Trying to clean up old dockers."
  docker system prune -a -f
 }
 clone_to_tmp() {
@@ -63,7 +120,7 @@ clone_to_tmp() {
 copy_new_files() {
  # Copy new files over to the salt dir
-  cd /tmp/sogh/securityonion
+  cd $UPDATE_DIR
  rsync -a salt $DEFAULT_SALT_DIR/
  rsync -a pillar $DEFAULT_SALT_DIR/
  chown -R socore:socore $DEFAULT_SALT_DIR/
@@ -112,7 +169,7 @@ masterunlock() {
 playbook() {
  echo "Applying playbook settings"
  if [[ "$INSTALLEDVERSION" =~ rc.1 ]]; then
-    salt-call state.apply playbook.db_init
+    salt-call state.apply playbook.OLD_db_init
    rm -f /opt/so/rules/elastalert/playbook/*.yaml
    so-playbook-ruleupdate >> /root/soup_playbook_rule_update.log 2>&1 &
  fi
@@ -124,6 +181,7 @@ pillar_changes() {
    [[ "$INSTALLEDVERSION" =~ rc.1 ]] && rc1_to_rc2
    [[ "$INSTALLEDVERSION" =~ rc.2 ]] && rc2_to_rc3
    [[ "$INSTALLEDVERSION" =~ rc.3 ]] && rc3_to_2.3.0
 }
@@ -184,6 +242,28 @@ rc2_to_rc3() {
  # Enable Strelka Rules
  sed -i "/  rules:/c\  rules: 1" /opt/so/saltstack/local/pillar/global.sls
  INSTALLEDVERSION=rc.3
 }
 rc3_to_2.3.0() {
  # Fix Tab Complete
  if [ ! -f /etc/profile.d/securityonion.sh ]; then
    echo "complete -cf sudo" > /etc/profile.d/securityonion.sh
  fi
  {
      echo "redis_settings:"
      echo "  redis_maxmemory: 827"
      echo "playbook:"
      echo "  api_key: de6639318502476f2fa5aa06f43f51fb389a3d7f" 
  } >> /opt/so/saltstack/local/pillar/global.sls
  sed -i 's/playbook:/playbook_db:/' /opt/so/saltstack/local/pillar/secrets.sls
  {
    echo "playbook_admin: $(tr -dc 'a-zA-Z0-9' < /dev/urandom | fold -w 20 | head -n 1)"
    echo "playbook_automation: $(tr -dc 'a-zA-Z0-9' < /dev/urandom | fold -w 20 | head -n 1)"
  } >> /opt/so/saltstack/local/pillar/secrets.sls
 }
 space_check() {
@@ -198,7 +278,33 @@ space_check() {
 }
 unmount_update() {
  cd /tmp
  umount /tmp/soagupdate
 }
 update_centos_repo() {
  # Update the files in the repo
  echo "Syncing new updates to /nsm/repo"
  rsync -a $AGDOCKER/repo /nsm/repo
  echo "Creating repo"
  createrepo /nsm/repo
 }
 update_dockers() {
  if [ $is_airgap -eq 0 ]; then
    # Let's copy the tarball
    if [ ! -f $AGDOCKER/registry.tar ]; then
      echo "Unable to locate registry. Exiting"
      exit 0
    else
      echo "Stopping the registry docker"
      docker stop so-dockerregistry
      docker rm so-dockerregistry
      echo "Copying the new dockers over"
      tar xvf $AGDOCKER/registry.tar -C /nsm/docker-registry/docker
    fi
  else
    # List all the containers
    if [ $MANAGERCHECK == 'so-import' ]; then
      TRUSTED_CONTAINERS=( \
@@ -263,7 +369,7 @@ update_dockers() {
      "so-telegraf" \
      "so-zeek" )
    fi
-
+    
 # Download the containers from the interwebs
    for i in "${TRUSTED_CONTAINERS[@]}"
    do
@@ -274,6 +380,13 @@ update_dockers() {
      docker tag $IMAGEREPO/$i:$NEWVERSION $HOSTNAME:5000/$IMAGEREPO/$i:$NEWVERSION
      docker push $HOSTNAME:5000/$IMAGEREPO/$i:$NEWVERSION 
    done
  fi
    # Cleanup on Aisle 4
    clean_dockers
    echo "Add Registry back if airgap"
    if [ $is_airgap -eq 0 ]; then
      docker load -i $AGDOCKER/registry_image.tar
    fi 
 }
@@ -334,7 +447,7 @@ upgrade_check_salt() {
 verify_latest_update_script() {
    # Check to see if the update scripts match. If not run the new one.
    CURRENTSOUP=$(md5sum /opt/so/saltstack/default/salt/common/tools/sbin/soup | awk '{print $1}')
-    GITSOUP=$(md5sum /tmp/sogh/securityonion/salt/common/tools/sbin/soup | awk '{print $1}')
+    GITSOUP=$(md5sum $UPDATE_DIR/salt/common/tools/sbin/soup | awk '{print $1}')
    if [[ "$CURRENTSOUP" == "$GITSOUP" ]]; then
      echo "This version of the soup script is up to date. Proceeding."
    else
@@ -366,12 +479,20 @@ done
 echo "Checking to see if this is a manager."
 echo ""
 manager_check
 echo "Checking to see if this is an airgap install"
 echo ""
 check_airgap
 echo "Found that Security Onion $INSTALLEDVERSION is currently installed."
 echo ""
 detect_os
 echo ""
-echo "Cloning Security Onion github repo into $UPDATE_DIR."
+if [ $is_airgap -eq 0 ]; then
-clone_to_tmp
+  # Let's mount the ISO since this is airgap
  airgap_mounted
 else
  echo "Cloning Security Onion github repo into $UPDATE_DIR."
  clone_to_tmp
 fi
 echo ""
 echo "Verifying we have the latest soup script."
 verify_latest_update_script
@@ -402,6 +523,11 @@ echo ""
 echo "Updating dockers to $NEWVERSION."
 update_dockers
 # Only update the repo if its airgap
 if [ $is_airgap -eq 0 ]; then
 update_centos_repo
 fi
 echo ""
 echo "Copying new Security Onion code from $UPDATE_DIR to $DEFAULT_SALT_DIR."
 copy_new_files
@@ -433,6 +559,7 @@ echo "Starting Salt Master service."
 systemctl start salt-master
 highstate
 playbook
 unmount_update
 SALTUPGRADED="True"
 if [[ "$SALTUPGRADED" == "True" ]]; then
@@ -1,7 +1,7 @@
 #!/bin/bash
-{%- if grains['role'] in ['so-node', 'so-searchnode', 'so-heavynode'] %}
+{%- if grains['role'] in ['so-node', 'so-heavynode'] %}
  {%- set ELASTICSEARCH_HOST = salt['pillar.get']('elasticsearch:mainip', '') -%}  
  {%- set ELASTICSEARCH_PORT = salt['pillar.get']('elasticsearch:es_port', '') -%}
  {%- set LOG_SIZE_LIMIT = salt['pillar.get']('elasticsearch:log_size_limit', '') -%}
@@ -11,7 +11,7 @@
  {%- set LOG_SIZE_LIMIT = salt['pillar.get']('manager:log_size_limit', '') -%}
 {%- endif -%}
-# Copyright 2014,2015,2016,2017,2018 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -33,20 +33,32 @@ LOG="/opt/so/log/curator/so-curator-closed-delete.log"
 # Check for 2 conditions:
 # 1. Are Elasticsearch indices using more disk space than LOG_SIZE_LIMIT?
-# 2. Are there any closed logstash-, or so- indices that we can delete?
+# 2. Are there any closed logstash- or so- indices that we can delete?
 # If both conditions are true, keep on looping until one of the conditions is false.
 while [[ $(du -hs --block-size=1GB /nsm/elasticsearch/nodes | awk '{print $1}' ) -gt "{{LOG_SIZE_LIMIT}}" ]] &&
-curl -s {{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/_cat/indices | grep -E "^      close (logstash-|so-)" > /dev/null; do
+{% if grains['role'] in ['so-node','so-heavynode'] %}
 curl -s -k https://{{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/_cat/indices | grep -E " close (logstash-|so-)" > /dev/null; do
 {% else %}
 curl -s {{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/_cat/indices | grep -E " close (logstash-|so-)" > /dev/null; do
 {% endif %}
 	# We need to determine OLDEST_INDEX.
 	# First, get the list of closed indices that are prefixed with "logstash-" or "so-".
 	# For example: logstash-ids-YYYY.MM.DD
 	# Then, sort by date by telling sort to use hyphen as delimiter and then sort on the third field.
 	# Finally, select the first entry in that sorted list.
-	OLDEST_INDEX=$(curl -s {{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/_cat/indices | grep -E "^      close (logstash-|so-)" | awk '{print $2}' | sort -t- -k3 | head -1)
+	{% if grains['role'] in ['so-node','so-heavynode'] %}
-
+	OLDEST_INDEX=$(curl -s -k https://{{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/_cat/indices | grep -E " close (logstash-|so-)" | awk '{print $2}' | sort -t- -k3 | head -1)
 	{% else %}
 	OLDEST_INDEX=$(curl -s {{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/_cat/indices | grep -E " close (logstash-|so-)" | awk '{print $2}' | sort -t- -k3 | head -1)
 	{% endif %}
 	# Now that we've determined OLDEST_INDEX, ask Elasticsearch to delete it.
 	{% if grains['role'] in ['so-node','so-heavynode'] %}
 	curl -XDELETE -k https://{{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/${OLDEST_INDEX}
 	{% else %}
 	curl -XDELETE {{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/${OLDEST_INDEX}
 	{% endif %}
 	# Finally, write a log entry that says we deleted it.
 	echo "$(date) - Used disk space exceeds LOG_SIZE_LIMIT ({{LOG_SIZE_LIMIT}} GB) - Index ${OLDEST_INDEX} deleted ..." >> ${LOG}
@@ -12,11 +12,11 @@ client:
    - {{elasticsearch}}
  port: 9200
  url_prefix:
-  use_ssl: False
+{% if grains['role'] in ['so-node', 'so-heavynode'] %}  use_ssl: True{% else %}  use_ssl: False{% endif %}
  certificate:
  client_cert:
  client_key:
-  ssl_no_validate: False
+{% if grains['role'] in ['so-node', 'so-heavynode'] %}  ssl_no_validate: True{% else %}  ssl_no_validate: False{% endif %}
  http_auth:
  timeout: 30
  master_only: False
@@ -0,0 +1,45 @@
 {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %}
 {% set MANAGER = salt['grains.get']('master') %}
 {% set OLDVERSIONS = ['2.0.0-rc.1','2.0.1-rc.1','2.0.2-rc.1','2.0.3-rc.1','2.1.0-rc.2','2.2.0-rc.3']%}
 {% for VERSION in OLDVERSIONS %}
 remove_images_{{ VERSION }}:
  docker_image.absent:
    - force: True
    - images:
      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-acng:{{ VERSION }}'
      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-thehive-cortex:{{ VERSION }}'
      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-curator:{{ VERSION }}'
      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-domainstats:{{ VERSION }}'
      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-elastalert:{{ VERSION }}'
      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-elasticsearch:{{ VERSION }}'
      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-filebeat:{{ VERSION }}'
      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-fleet:{{ VERSION }}'
      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-fleet-launcher:{{ VERSION }}'
      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-freqserver:{{ VERSION }}'
      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-grafana:{{ VERSION }}'
      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-idstools:{{ VERSION }}'
      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-influxdb:{{ VERSION }}'
      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-kibana:{{ VERSION }}'
      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-kratos:{{ VERSION }}'
      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-logstash:{{ VERSION }}'
      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-minio:{{ VERSION }}'
      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-mysql:{{ VERSION }}'
      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-nginx:{{ VERSION }}'
      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-pcaptools:{{ VERSION }}'
      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-playbook:{{ VERSION }}'
      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-redis:{{ VERSION }}'
      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-soc:{{ VERSION }}'
      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-soctopus:{{ VERSION }}'
      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-steno:{{ VERSION }}'
      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-strelka-frontend:{{ VERSION }}'
      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-strelka-manager:{{ VERSION }}'
      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-strelka-backend:{{ VERSION }}'
      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-strelka-filestream:{{ VERSION }}'
      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-suricata:{{ VERSION }}'
      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-telegraf:{{ VERSION }}'
      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-thehive:{{ VERSION }}'
      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-thehive-es:{{ VERSION }}'
      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-wazuh:{{ VERSION }}'
      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-zeek:{{ VERSION }}'
 {% endfor %}
@@ -16,7 +16,7 @@ class PlaybookESAlerter(Alerter):
            today = strftime("%Y.%m.%d", gmtime())
            timestamp = strftime("%Y-%m-%d"'T'"%H:%M:%S", gmtime())
            headers = {"Content-Type": "application/json"}
-            payload = {"rule.name": self.rule['play_title'],"event.severity": self.rule['event.severity'],"kibana_pivot": self.rule['kibana_pivot'],"soc_pivot": self.rule['soc_pivot'],"event.module": self.rule['event.module'],"event.dataset": self.rule['event.dataset'],"play_url": self.rule['play_url'],"sigma_level": self.rule['sigma_level'],"rule.category": self.rule['rule.category'],"event_data": match, "@timestamp": timestamp}
+            payload = {"rule": { "name": self.rule['play_title'],"uuid": self.rule['play_id'],"category": self.rule['rule.category']},"event":{ "severity": self.rule['event.severity'],"module": self.rule['event.module'],"dataset": self.rule['event.dataset'],"severity_label": self.rule['sigma_level']},"kibana_pivot": self.rule['kibana_pivot'],"soc_pivot": self.rule['soc_pivot'],"play_url": self.rule['play_url'],"sigma_level": self.rule['sigma_level'],"event_data": match, "@timestamp": timestamp}
            url = f"http://{self.rule['elasticsearch_host']}/so-playbook-alerts-{today}/_doc/"
            requests.post(url, data=json.dumps(payload), headers=headers, verify=False)
@@ -1,47 +0,0 @@
 {% set es = salt['pillar.get']('global:managerip', '') %}
 {% set hivehost = salt['pillar.get']('global:managerip', '') %}
 {% set hivekey = salt['pillar.get']('global:hivekey', '') %}
 {% set MANAGER = salt['pillar.get']('global:url_base', '') %}
 # Elastalert rule to forward Suricata alerts from Security Onion to a specified TheHive instance.
 #
 es_host: {{es}}
 es_port: 9200
 name: Suricata-Alert
 type: any
 index: "*:so-ids-*"
 buffer_time:
    minutes: 5
 query_key: ["rule.uuid","source.ip","destination.ip"]
 realert:
    days: 1
 filter:
 - query:
   query_string:
      query: "event.module: suricata AND rule.severity:(1 OR 2)"
 alert: hivealerter
 hive_connection:
  hive_host: http://{{hivehost}}
  hive_port: 9000/thehive
  hive_apikey: {{hivekey}}
 hive_proxies:
  http: ''
  https: ''
 hive_alert_config:
  title: '{match[rule][name]}'
  type: 'NIDS'
  source: 'SecurityOnion'
  description: "`SOC Hunt Pivot:` \n\n <https://{{MANAGER}}/#/hunt?q=network.community_id%3A%20%20%22{match[network][community_id]}%22%20%7C%20groupby%20source.ip%20destination.ip,event.module,%20event.dataset>  \n\n `Kibana Dashboard Pivot:` \n\n <https://{{MANAGER}}/kibana/app/kibana#/dashboard/30d0ac90-729f-11ea-8dd2-9d8795a1200b?_g=(filters:!(('$state':(store:globalState),meta:(alias:!n,disabled:!f,index:'2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29',key:network.community_id,negate:!f,params:(query:'{match[network][community_id]}'),type:phrase),query:(match_phrase:(network.community_id:'{match[network][community_id]}')))),refreshInterval:(pause:!t,value:0),time:(from:now-7d,to:now))>  \n\n `IPs: `{match[source][ip]}:{match[source][port]} --> {match[destination][ip]}:{match[destination][port]} \n\n `Signature:`{match[rule][rule]}"
  severity: 2
  tags: ['{match[rule][uuid]}','{match[source][ip]}','{match[destination][ip]}']
  tlp: 3
  status: 'New'
  follow: True
 hive_observable_data_mapping:
  - ip: '{match[source][ip]}'
  - ip: '{match[destination][ip]}'
@@ -1,45 +0,0 @@
 {% set es = salt['pillar.get']('global:managerip', '') %}
 {% set hivehost = salt['pillar.get']('global:managerip', '') %}
 {% set hivekey = salt['pillar.get']('global:hivekey', '') %}
 {% set MANAGER = salt['pillar.get']('global:url_base', '') %}
 # Elastalert rule to forward high level Wazuh alerts from Security Onion to a specified TheHive instance.
 #
 es_host: {{es}}
 es_port: 9200
 name: Wazuh-Alert
 type: any
 index: "*:so-ossec-*"
 buffer_time:
    minutes: 5
 realert:
    days: 1
 filter:
 - query:
   query_string:
      query: "event.module: ossec AND rule.level>=8"
 alert: hivealerter
 hive_connection:
  hive_host: http://{{hivehost}}
  hive_port: 9000/thehive
  hive_apikey: {{hivekey}}
 hive_proxies:
  http: ''
  https: ''
 hive_alert_config:
  title: '{match[rule][name]}'
  type: 'wazuh'
  source: 'SecurityOnion'
  description: "`SOC Hunt Pivot:` \n\n <https://{{MANAGER}}/#/hunt?q=event.module%3A%20ossec%20AND%20rule.id%3A{match[rule][id]}%20%7C%20groupby%20host.name%20rule.name>  \n\n `Kibana Dashboard Pivot:` \n\n <https://{{MANAGER}}/kibana/app/kibana#/dashboard/ed6f7e20-e060-11e9-8f0c-2ddbf5ed9290?_g=(refreshInterval:(display:Off,pause:!f,value:0),time:(from:now-24h,mode:quick,to:now))&_a=(columns:!(_source),index:'*:logstash-*',interval:auto,query:(query_string:(analyze_wildcard:!t,query:'sid:')),sort:!('@timestamp',desc))>"
  severity: 2
  tags: ['{match[rule][id]}','{match[host][name]}']
  tlp: 3
  status: 'New'
  follow: True
 hive_observable_data_mapping:
  - other: '{match[host][name]}'
@@ -91,14 +91,6 @@ elastasomodulesync:
    - group: 933
    - makedirs: True
 elastarulesync:
  file.recurse:
    - name: /opt/so/rules/elastalert
    - source: salt://elastalert/files/rules/so
    - user: 933
    - group: 933
    - template: jinja
 elastaconf:
  file.managed:
    - name: /opt/so/conf/elastalert/elastalert_config.yaml
@@ -137,4 +129,4 @@ elastalert_state_not_allowed:
  test.fail_without_changes:
    - name: elastalert_state_not_allowed
-{% endif %}
+{% endif %}
@@ -21,40 +21,30 @@
 		"properties": ["ip", "country_iso_code", "country_name", "continent_name", "region_iso_code", "region_name", "city_name", "timezone", "location"]
 	}
    },
-    { 
+    { "set":             { "if": "ctx.event?.severity == 1",   "field": "event.severity_label", "value": "low", "override": true }  },
-	"split":  {
+    { "set":             { "if": "ctx.event?.severity == 2",   "field": "event.severity_label", "value": "medium", "override": true }  },
-		"field": "_index",
+    { "set":             { "if": "ctx.event?.severity == 3",   "field": "event.severity_label", "value": "high", "override": true }  },
-		"target_field": "index_name_prefix",
+    { "set":             { "if": "ctx.event?.severity == 4",   "field": "event.severity_label", "value": "critical", "override": true }  },
-		"separator": "-"
+    { "rename":          { "field": "fields.category",              "target_field": "event.category",                  "ignore_failure": true, "ignore_missing": true  } },
-	}
+    { "rename":          { "field": "fields.module",              "target_field": "event.module",                  "ignore_failure": true, "ignore_missing": true  } },
-    },
+    { "rename":          { "field": "module",              "target_field": "event.module",                  "ignore_failure": true, "ignore_missing": true  } },
-    { 
+    { "rename":          { "field": "dataset",             "target_field": "event.dataset",                "ignore_failure": true, "ignore_missing": true  } },
-	"date_index_name": {
+    { "rename":          { "field": "category",            "target_field": "event.category",              "ignore_failure": true, "ignore_missing": true  } },
-        	"field": "@timestamp",
+    { "rename":          { "field": "message2.community_id", "target_field": "network.community_id",  "ignore_failure": true,  "ignore_missing": true  } },
-	        "index_name_prefix": "{{index_name_prefix.0}}-{{index_name_prefix.1}}-",
+    { "lowercase":       { "field": "event.dataset", "ignore_failure": true,  "ignore_missing": true  } },
        	"date_rounding": "d",
 	        "ignore_failure": true,
        	"index_name_format": "yyyy.MM.dd"
        }
    },  
    { "set":         { "if": "ctx.event?.severity == 1",   "field": "event.severity_label", "value": "low", "override": true }  },
    { "set":         { "if": "ctx.event?.severity == 2",   "field": "event.severity_label", "value": "medium", "override": true }  },
    { "set":         { "if": "ctx.event?.severity == 3",   "field": "event.severity_label", "value": "high", "override": true }  },
    { "set":         { "if": "ctx.event?.severity == 4",   "field": "event.severity_label", "value": "critical", "override": true }  },
    { "rename":         { "field": "module",              "target_field": "event.module",                  "ignore_failure": true, "ignore_missing": true  } },
    { "rename":         { "field": "dataset",             "target_field": "event.dataset",                "ignore_failure": true, "ignore_missing": true  } },
    { "rename":         { "field": "category",            "target_field": "event.category",              "ignore_failure": true, "ignore_missing": true  } },
    { "rename":         { "field": "message2.community_id", "target_field": "network.community_id",  "ignore_failure": true,  "ignore_missing": true  } },
    { "lowercase":         { "field": "event.dataset", "ignore_failure": true,  "ignore_missing": true  } },    
    { "convert":         { "field": "destination.port", "type": "integer",  "ignore_failure": true,  "ignore_missing": true  } },
    { "convert":         { "field": "source.port", "type": "integer",  "ignore_failure": true,  "ignore_missing": true  } },
    { "convert":         { "field": "log.id.uid", "type": "string",  "ignore_failure": true,  "ignore_missing": true  } },
    { "convert":         { "field": "agent.id", "type": "string",  "ignore_failure": true,  "ignore_missing": true  } },
    { "convert":         { "field": "event.severity", "type": "integer",  "ignore_failure": true,  "ignore_missing": true  } },
-    { 
+    { "remove":          { "field": [ "message2", "type", "fields", "category", "module", "dataset" ], "ignore_missing": true, "ignore_failure": true } }, 
-        "remove": {
+    {
-                "field": [ "index_name_prefix", "message2", "type" ],
+        "date_index_name": {
-                "ignore_failure": true
+                "field": "@timestamp",
                "index_name_prefix": "{{ _index }}-",
                "date_rounding": "d",
                "ignore_failure": true,
                "index_name_format": "yyyy.MM.dd"
        }
    }
  ]
@@ -0,0 +1,17 @@
 {
  "description" : "common.nids",
  "processors" : [
    { "convert":        { "if": "ctx.rule.uuid != null",                "field": "rule.uuid",                   "type": "integer"                                                                               } },
    { "set":            { "if": "ctx.rule?.uuid < 1000000",             "field": "rule.reference", "value": "https://www.snort.org/search?query={{rule.gid}}-{{rule.uuid}}"     } },
    { "set":            { "if": "ctx.rule?.uuid > 1999999",             "field": "rule.reference", "value": "https://doc.emergingthreats.net/{{rule.uuid}}"                     } },
    { "convert":        { "if": "ctx.rule.uuid != null",                        "field": "rule.uuid",                   "type": "string"                                                                               } },
    { "dissect":        { "if": "ctx.rule.name != null",                        "field": "rule.name",                   "pattern" : "%{rule_type} %{rest_of_rulename} ",                                "ignore_failure": true  } },
    { "set":            { "if": "ctx.rule_type == 'GPL'",       "field": "rule_ruleset", "value": "Snort GPL"                                                   } },
    { "set":            { "if": "ctx.rule_type == 'ET'",        "field": "rule.ruleset", "value": "Emerging Threats"                                            } },
    { "set":            { "if": "ctx.rule.severity == 3",   "field": "event.severity", "value": 1, "override": true }  },
    { "set":            { "if": "ctx.rule.severity == 2",   "field": "event.severity", "value": 2, "override": true }  },
    { "set":            { "if": "ctx.rule.severity == 1",   "field": "event.severity", "value": 3, "override": true }  },
    { "remove":         { "field": ["rule_type", "rest_of_rulename"],     "ignore_failure": true                                                                  } },
    { "pipeline":       { "name": "common"                                                                                                                      } }
  ]
 }
@@ -1,17 +0,0 @@
 {
  "description" : "common_nids",
  "processors" : [
    { "convert": 	{ "field": "sid", 			"type": "integer" 										} },
    { "set": 		{ "if": "ctx.sid < 1000000",		"field": "signature_info", "value": "https://www.snort.org/search?query={{gid}}-{{sid}}"	} },
    { "set": 		{ "if": "ctx.sid > 1999999",		"field": "signature_info", "value": "https://doc.emergingthreats.net/{{sid}}"			} },
    { "remove":		{ "if": "ctx.sid > 2999999",		"field": "signature_info" 									} },
    { "set": 		{ "if": "ctx.priority == '1'",		"field": "severity", "value": "High"	 							} },
    { "set": 		{ "if": "ctx.priority == '2'",		"field": "severity", "value": "Medium" 								} },
    { "set": 		{ "if": "ctx.priority == '3'",		"field": "severity", "value": "Low" 								} },
    { "dissect": 	{ "field": "alert",			"pattern" : "%{rule_type} %{category} ",				"ignore_failure": true 	} },
    { "set": 		{ "if": "ctx.rule_type == 'GPL'",	"field": "rule_type", "value": "Snort GPL"							} },
    { "set": 		{ "if": "ctx.rule_type == 'ET'",	"field": "rule_type", "value": "Emerging Threats"						} },
    { "lowercase": 	{ "field": "category",												"ignore_failure": true	} },
    { "pipeline":       { "name": "common"															} }
  ]
 }
@@ -0,0 +1,59 @@
 {
  "description" : "filterlog",
  "processors" : [
      {
        "dissect": {
                "field": "real_message",
                "pattern" : "%{rule.uuid},%{rule.sub_uuid},%{firewall.anchor},%{firewall.tracker_id},%{interface.name},%{rule.reason},%{rule.action},%{network.direction},%{ip.version},%{firewall.sub_message}",
                "on_failure" : [ {"set" : {"field" : "error.message","value" : "{{ _ingest.on_failure_message }}"}}]
        }
     },
     {
        "dissect": {
                "if": "ctx.ip.version == '4'",
                "field": "firewall.sub_message",
                "pattern" : "%{ip.tos},%{ip.ecn},%{ip.ttl},%{ip.id},%{ip.offset},%{ip.flags},%{network.transport_id},%{network.transport},%{data.length},%{source.ip},%{destination.ip},%{ip_sub_msg}",
                "on_failure" : [ {"set" : {"field" : "error.message","value" : "{{ _ingest.on_failure_message }}"}}]
       }
     },
     {
        "dissect": {
                "if": "ctx.ip?.version == '6'",
                "field": "firewall.sub_message",
                "pattern" : "%{network.class},%{network.flow_label},%{network.hop_limit},%{network.transport},%{network.transport_id},%{data.length},%{source.ip},%{destination.ip},%{ip_sub_msg}",
                "on_failure" : [ {"set" : {"field" : "error.message","value" : "{{ _ingest.on_failure_message }}"}}]
        }
     },
     {
        "dissect": {
                "if": "ctx.network?.transport == 'tcp'",
                "field": "ip_sub_msg",
                "pattern" : "%{source.port},%{destination.port},%{data.length},%{tcp.flags},",
                "on_failure" : [ {"set" : {"field" : "error.message","value" : "{{ _ingest.on_failure_message }}"}}]
        }
     },
     {
        "dissect": {
                "if": "ctx.network?.transport == 'udp'",
                "field": "ip_sub_msg",
                "pattern" : "%{source.port},%{destination.port},%{data.length}",
                "on_failure" : [ {"set" : {"field" : "error.message","value" : "{{ _ingest.on_failure_message }}"}}]
        }
     },
     {
        "split": {
                "if": "ctx.ip.version =='6' && ctx.network?.transport == 'Options'",
                "field": "ip_sub_msg",
                "target_field": "ip.options",
                "separator" : ",",
                "on_failure" : [ {"set" : {"field" : "error.message","value" : "{{ _ingest.on_failure_message }}"}}]
        }
     },
     { "set":         { "field": "_index",        "value": "so-firewall", "override": true           } },
     { "set":         { "if": "ctx.network?.transport_id == '0'", "field": "network.transport",        "value": "icmp", "override": true           } },
     {"community_id": { "if": "ctx.network?.transport != null", "field":["source.ip","source.port","destination.ip","destination.port","network.transport"],"target_field":"network.community_id"}},
     { "set":         { "field": "module",        "value": "pfsense", "override": true           } },
     { "set":         { "field": "dataset",        "value": "firewall", "override": true           } },
     { "remove":      { "field": ["real_message", "ip_sub_msg", "firewall.sub_message"],     "ignore_failure": true  } }
  ]
 }
@@ -1,52 +1,69 @@
 {
  "description" : "ossec",
  "processors" : [
-    { "json":		{ "field": "message",					"target_field": "message2",		"ignore_failure": true	} },
+    { "json":           { "field": "message",                                   "target_field": "message2",             "ignore_failure": true  } },
-    { "rename": 	{ "field": "message2.agent", 				"target_field": "agent",		"ignore_missing": true 	} },
+    { "remove": { "field": [ "agent" ], "ignore_missing": true, "ignore_failure": true } },
-    { "rename": 	{ "field": "message2.data", 				"target_field": "data",			"ignore_missing": true 	} },
+    { "rename":         { "field": "message2.agent",                            "target_field": "agent",                "ignore_missing": true  } },
-    { "rename": 	{ "field": "message2.decoder", 				"target_field": "decoder",		"ignore_missing": true 	} },
+    { "rename":         { "field": "message2.data",                             "target_field": "data",                 "ignore_missing": true  } },
-    { "rename": 	{ "field": "message2.full_log", 			"target_field": "full_log",		"ignore_missing": true 	} },
+    { "rename":         { "field": "message2.decoder",                          "target_field": "decoder",              "ignore_missing": true  } },
-    { "rename": 	{ "field": "message2.id", 				"target_field": "log.id.id",			"ignore_missing": true 	} },
+    { "rename":         { "field": "message2.full_log",                         "target_field": "log.full",             "ignore_missing": true  } },
-    { "rename": 	{ "field": "message2.location", 			"target_field": "location",		"ignore_missing": true 	} },
+    { "rename":         { "field": "message2.id",                               "target_field": "log.id.id",                    "ignore_missing": true  } },
-    { "rename": 	{ "field": "message2.manager", 				"target_field": "manager",		"ignore_missing": true 	} },
+    { "rename":         { "field": "message2.location",                         "target_field": "log.location",         "ignore_missing": true  } },
-    { "rename": 	{ "field": "message2.predecoder", 			"target_field": "predecoder",		"ignore_missing": true 	} },
+    { "rename":         { "field": "message2.manager",                          "target_field": "manager",              "ignore_missing": true  } },
-    { "rename": 	{ "field": "message2.timestamp", 			"target_field": "timestamp",		"ignore_missing": true 	} },
+    { "rename":         { "field": "message2.predecoder",                       "target_field": "predecoder",           "ignore_missing": true  } },
-    { "rename": 	{ "field": "message2.rule", 				"target_field": "rule",		"ignore_missing": true 	} },
+    { "rename":         { "field": "message2.timestamp",                        "target_field": "event.timestamp",              "ignore_missing": true  } },
-    { "rename": 	{ "field": "data.command", 				"target_field": "command",		"ignore_missing": true 	} },
+    { "rename":         { "field": "message2.previous_log",                        "target_field": "log.previous_log",            "ignore_missing": true  } },
-    { "rename": 	{ "field": "data.dstip", 				"target_field": "destination.ip",	"ignore_missing": true 	} },
+    { "rename":         { "field": "message2.previous_output",                        "target_field": "log.previous_output",            "ignore_missing": true  } },
-    { "rename": 	{ "field": "data.dstport", 				"target_field": "destination.port",	"ignore_missing": true 	} },
+    { "rename":         { "field": "message2.rule",                             "target_field": "rule",         "ignore_missing": true  } },
-    { "rename": 	{ "field": "data.dstuser", 				"target_field": "user.escalated",	"ignore_missing": true 	} },
+    { "rename":         { "field": "message2.syscheck",                         "target_field": "host.syscheck",        "ignore_missing": true  } },
-    { "rename": 	{ "field": "data.srcip", 				"target_field": "source.ip",		"ignore_missing": true 	} },
+    { "rename":         { "field": "data.command",                              "target_field": "process.command_line",         "ignore_missing": true  } },
-    { "rename": 	{ "field": "data.srcuser", 				"target_field": "source.user",		"ignore_missing": true 	} },
+    { "rename":         { "field": "data.dstip",                                "target_field": "destination.ip",       "ignore_missing": true  } },
-    { "rename": 	{ "field": "data.win.eventdata.destinationHostname", 	"target_field": "destination.hostname",	"ignore_missing": true 	} },
+    { "rename":         { "field": "data.dstport",                              "target_field": "destination.port",     "ignore_missing": true  } },
-    { "rename": 	{ "field": "data.win.eventdata.destinationIp", 		"target_field": "destination.ip",	"ignore_missing": true 	} },
+    { "rename":         { "field": "data.dstuser",                              "target_field": "user.escalated",       "ignore_missing": true  } },
-    { "rename": 	{ "field": "data.win.eventdata.destinationPort", 	"target_field": "destination.port",	"ignore_missing": true 	} },
+    { "rename":         { "field": "data.srcip",                                "target_field": "source.ip",            "ignore_missing": true  } },
-    { "rename": 	{ "field": "data.win.eventdata.image", 			"target_field": "image_path",		"ignore_missing": true 	} },
+    { "rename":         { "field": "data.process",                              "target_field": "process",              "ignore_missing": true  } },
-    { "rename": 	{ "field": "data.win.eventdata.parentImage", 		"target_field": "parent_image_path",	"ignore_missing": true 	} },
+    { "rename":         { "field": "data.program",                              "target_field": "program",              "ignore_missing": true  } },
-    { "rename": 	{ "field": "data.win.eventdata.sourceHostname", 	"target_field": "source.hostname",	"ignore_missing": true 	} },
+    { "rename":         { "field": "data.win.eventdata.destinationHostname",    "target_field": "destination.hostname", "ignore_missing": true  } },
-    { "rename": 	{ "field": "data.win.eventdata.sourceIp", 		"target_field": "source_ip",		"ignore_missing": true 	} },
+    { "rename":         { "field": "data.win.eventdata.destinationIp",          "target_field": "destination.ip",       "ignore_missing": true  } },
-    { "rename": 	{ "field": "data.win.eventdata.sourcePort", 		"target_field": "source.port",		"ignore_missing": true 	} },
+    { "rename":         { "field": "data.win.eventdata.destinationPort",        "target_field": "destination.port",     "ignore_missing": true  } },
-    { "rename": 	{ "field": "data.win.eventdata.targetFilename",		"target_field": "file.target",	"ignore_missing": true 	} },
+    { "rename":         { "field": "data.win.eventdata.sourceHostname",         "target_field": "source.hostname",      "ignore_missing": true  } },
-    { "rename": 	{ "field": "data.win.eventdata.user", 			"target_field": "user.name",		"ignore_missing": true 	} },
+    { "rename":         { "field": "data.win.eventdata.sourceIp",               "target_field": "source_ip",            "ignore_missing": true  } },
-    { "rename": 	{ "field": "data.win.system.eventID", 			"target_field": "event.code",		"ignore_missing": true 	} },
+    { "rename":         { "field": "data.win.eventdata.sourcePort",             "target_field": "source.port",          "ignore_missing": true  } },
-    { "rename": 	{ "field": "predecoder.program_name", 			"target_field": "process.name",		"ignore_missing": true 	} },
+    { "rename":         { "field": "data.win.eventdata.targetFilename",         "target_field": "file.target",  "ignore_missing": true  } },
-    { "set": 		{ "if": "ctx.rule.level == 1", 		"field": "rule.category", "value": "None" 				} },
+    { "rename":         { "field": "data.win.eventdata.user",                   "target_field": "user.name",            "ignore_missing": true  } },
-    { "set": 		{ "if": "ctx.rule.level == 2", 		"field": "rule.category", "value": "System low priority notification" 	} },
+    { "rename":         { "field": "data.win.system",                   "target_field": "winlog",           "ignore_missing": true  } },
-    { "set": 		{ "if": "ctx.rule.level == 3", 		"field": "rule.category", "value": "Successful/authorized event" 	} },
+    { "rename":         { "field": "data.win.eventdata",                   "target_field": "winlog.event_data",           "ignore_missing": true  } },
-    { "set": 		{ "if": "ctx.rule.level == 4", 		"field": "rule.category", "value": "System low priority error" 	} },
+    { "rename":         { "field": "winlog.eventID",                   "target_field": "winlog.event_id",           "ignore_missing": true  } },
-    { "set": 		{ "if": "ctx.rule.level == 5", 		"field": "rule.category", "value": "User generated error" 		} },
+    { "rename":         { "field": "predecoder.program_name",                   "target_field": "process.name",         "ignore_missing": true  } },
-    { "set": 		{ "if": "ctx.rule.level == 6", 		"field": "rule.category", "value": "Low relevance attack" 		} },
+    { "rename":         { "field": "decoder.name",                   "target_field": "event.dataset",         "ignore_missing": true  } },
-    { "set": 		{ "if": "ctx.rule.level == 7", 		"field": "rule.category", "value": "\"Bad word\" matching" 		} },
+    { "rename":         { "field": "rule.description",                   "target_field": "rule.name",         "ignore_missing": true  } },
-    { "set": 		{ "if": "ctx.rule.level == 8", 		"field": "rule.category", "value": "First time seen" 			} },
+    { "rename":         { "field": "rule.id",                            "target_field": "rule.uuid",         "ignore_missing": true  } },
-    { "set": 		{ "if": "ctx.rule.level == 9", 		"field": "rule.category", "value": "Error from invalid source" 	} },
+    { "set":            { "if": "ctx.rule != null && ctx.rule.level == 1",          "field": "rule.category", "value": "None"                               } },
-    { "set": 		{ "if": "ctx.rule.level == 10", 		"field": "rule.category", "value": "Multiple user generated errors" 	} },
+    { "set":            { "if": "ctx.rule != null && ctx.rule.level == 2",          "field": "rule.category", "value": "System low priority notification"   } },
-    { "set": 		{ "if": "ctx.rule.level == 11", 		"field": "rule.category", "value": "Integrity checking warning" 	} },
+    { "set":            { "if": "ctx.rule != null && ctx.rule.level == 3",          "field": "rule.category", "value": "Successful/authorized event"        } },
-    { "set": 		{ "if": "ctx.rule.level == 12", 		"field": "rule.category", "value": "High importance event" 		} },
+    { "set":            { "if": "ctx.rule != null && ctx.rule.level == 4",          "field": "rule.category", "value": "System low priority error"  } },
-    { "set": 		{ "if": "ctx.rule.level == 13", 		"field": "rule.category", "value": "Unusal error (high importance)" 	} },
+    { "set":            { "if": "ctx.rule != null && ctx.rule.level == 5",          "field": "rule.category", "value": "User generated error"               } },
-    { "set": 		{ "if": "ctx.rule.level == 14", 		"field": "rule.category", "value": "High importance security event" 	} },
+    { "set":            { "if": "ctx.rule != null && ctx.rule.level == 6",          "field": "rule.category", "value": "Low relevance attack"               } },
-    { "set": 		{ "if": "ctx.rule.level == 15", 		"field": "rule.category", "value": "Severe attack" 			} },
+    { "set":            { "if": "ctx.rule != null && ctx.rule.level == 7",          "field": "rule.category", "value": "\"Bad word\" matching"              } },
-    { "append": 	{ "if": "ctx.rule.level != null",		"field": "tags", "value": ["alert"] 					} },
+    { "set":            { "if": "ctx.rule != null && ctx.rule.level == 8",          "field": "rule.category", "value": "First time seen"                    } },
-    { "remove": { "field": [ "host", "predecoder", "decoder" ], "ignore_missing": true, "ignore_failure": false } },
+    { "set":            { "if": "ctx.rule != null && ctx.rule.level == 9",          "field": "rule.category", "value": "Error from invalid source"  } },
-    { "pipeline":	{ "name": "common" } }
+    { "set":            { "if": "ctx.rule != null && ctx.rule.level == 10",                 "field": "rule.category", "value": "Multiple user generated errors"     } },
    { "set":            { "if": "ctx.rule != null && ctx.rule.level == 11",                 "field": "rule.category", "value": "Integrity checking warning"         } },
    { "set":            { "if": "ctx.rule != null && ctx.rule.level == 12",                 "field": "rule.category", "value": "High importance event"              } },
    { "set":            { "if": "ctx.rule != null && ctx.rule.level == 13",                 "field": "rule.category", "value": "Unusal error (high importance)"     } },
    { "set":            { "if": "ctx.rule != null && ctx.rule.level == 14",                 "field": "rule.category", "value": "High importance security event"     } },
    { "set":            { "if": "ctx.rule != null && ctx.rule.level == 15",                 "field": "rule.category", "value": "Severe attack"                      } },
    { "set":         { "if": "ctx.rule != null && ctx.rule.level >= 1 && ctx.rule.level <=7",   "field": "event.severity", "value": 1, "override": true }  },
    { "set":         { "if": "ctx.rule != null && ctx.rule.level >= 8 && ctx.rule.level <=11",   "field": "event.severity", "value": 2, "override": true }  },
    { "set":         { "if": "ctx.rule != null && ctx.rule.level >= 12 && ctx.rule.level <=14",   "field": "event.severity", "value": 3, "override": true }  },
    { "set":         { "if": "ctx.rule != null && ctx.rule.level >= 15",   "field": "event.severity", "value": 4, "override": true }  },
    { "rename":         { "field": "rule.id",                            "target_field": "rule.uuid",         "ignore_missing": true  } },
    { "remove":         { "field": [ "predecoder" ],                                                          "ignore_failure": true  } },
    { "rename":         { "field": "fields.category",              "target_field": "event.category",                  "ignore_failure": true, "ignore_missing": true  } },
    { "rename":         { "field": "fields.module",              "target_field": "event.module",                  "ignore_failure": true, "ignore_missing": true  } },
    { "pipeline":      { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational'",   "name": "sysmon"  }  },
    { "pipeline":      { "if": "ctx.winlog?.channel != 'Microsoft-Windows-Sysmon/Operational'",  "name":"win.eventlogs" }  },
    { "set":           { "if": "ctx.containsKey('rule') && ctx.rule != null",   "field": "event.dataset", "value": "alert", "override": true }  },
    { "pipeline":       { "name": "common" } }    
  ]
 }
@@ -1,59 +0,0 @@
 {
  "description" : "ossec",
  "processors" : [
    { "json":           { "field": "message",                                   "target_field": "message2",             "ignore_failure": true  } },
    { "remove": { "field": [ "agent" ], "ignore_missing": true, "ignore_failure": false } },
    { "rename":         { "field": "message2.agent",                            "target_field": "agent",                "ignore_missing": true  } },
    { "rename":         { "field": "message2.data",                             "target_field": "data",                 "ignore_missing": true  } },
    { "rename":         { "field": "message2.decoder",                          "target_field": "decoder",              "ignore_missing": true  } },
    { "rename":         { "field": "message2.full_log",                         "target_field": "log.full",             "ignore_missing": true  } },
    { "rename":         { "field": "message2.id",                               "target_field": "log.id.id",                    "ignore_missing": true  } },
    { "rename":         { "field": "message2.location",                         "target_field": "log.location",         "ignore_missing": true  } },
    { "rename":         { "field": "message2.manager",                          "target_field": "manager",              "ignore_missing": true  } },
    { "rename":         { "field": "message2.predecoder",                       "target_field": "predecoder",           "ignore_missing": true  } },
    { "rename":         { "field": "message2.timestamp",                        "target_field": "event.timestamp",              "ignore_missing": true  } },
    { "rename":         { "field": "message2.previous_log",                        "target_field": "log.previous_log",            "ignore_missing": true  } },
    { "rename":         { "field": "message2.previous_output",                        "target_field": "log.previous_output",            "ignore_missing": true  } },
    { "rename":         { "field": "message2.rule",                             "target_field": "rule",         "ignore_missing": true  } },
    { "rename":         { "field": "message2.syscheck",                         "target_field": "host.syscheck",        "ignore_missing": true  } },
    { "rename":         { "field": "data.command",                              "target_field": "process.command_line",         "ignore_missing": true  } },
    { "rename":         { "field": "data.dstip",                                "target_field": "destination.ip",       "ignore_missing": true  } },
    { "rename":         { "field": "data.dstport",                              "target_field": "destination.port",     "ignore_missing": true  } },
    { "rename":         { "field": "data.dstuser",                              "target_field": "user.escalated",       "ignore_missing": true  } },
    { "rename":         { "field": "data.srcip",                                "target_field": "source.ip",            "ignore_missing": true  } },
    { "rename":         { "field": "data.win.eventdata.destinationHostname",    "target_field": "destination.hostname", "ignore_missing": true  } },
    { "rename":         { "field": "data.win.eventdata.destinationIp",          "target_field": "destination.ip",       "ignore_missing": true  } },
    { "rename":         { "field": "data.win.eventdata.destinationPort",        "target_field": "destination.port",     "ignore_missing": true  } },
    { "rename":         { "field": "data.win.eventdata.image",                  "target_field": "image_path",           "ignore_missing": true  } },
    { "rename":         { "field": "data.win.eventdata.parentImage",            "target_field": "parent_image_path",    "ignore_missing": true  } },
    { "rename":         { "field": "data.win.eventdata.sourceHostname",         "target_field": "source.hostname",      "ignore_missing": true  } },
    { "rename":         { "field": "data.win.eventdata.sourceIp",               "target_field": "source_ip",            "ignore_missing": true  } },
    { "rename":         { "field": "data.win.eventdata.sourcePort",             "target_field": "source.port",          "ignore_missing": true  } },
    { "rename":         { "field": "data.win.eventdata.targetFilename",         "target_field": "file.target",  "ignore_missing": true  } },
    { "rename":         { "field": "data.win.eventdata.user",                   "target_field": "user.name",            "ignore_missing": true  } },
    { "rename":         { "field": "data.win.system.eventID",                   "target_field": "event.code",           "ignore_missing": true  } },
    { "rename":         { "field": "predecoder.program_name",                   "target_field": "process.name",         "ignore_missing": true  } },
    { "rename":         { "field": "rule.description",                   "target_field": "rule.name",         "ignore_missing": true  } },
    { "set":            { "if": "ctx.rule.level == 1",          "field": "rule.category", "value": "None"                               } },
    { "set":            { "if": "ctx.rule.level == 2",          "field": "rule.category", "value": "System low priority notification"   } },
    { "set":            { "if": "ctx.rule.level == 3",          "field": "rule.category", "value": "Successful/authorized event"        } },
    { "set":            { "if": "ctx.rule.level == 4",          "field": "rule.category", "value": "System low priority error"  } },
    { "set":            { "if": "ctx.rule.level == 5",          "field": "rule.category", "value": "User generated error"               } },
    { "set":            { "if": "ctx.rule.level == 6",          "field": "rule.category", "value": "Low relevance attack"               } },
    { "set":            { "if": "ctx.rule.level == 7",          "field": "rule.category", "value": "\"Bad word\" matching"              } },
    { "set":            { "if": "ctx.rule.level == 8",          "field": "rule.category", "value": "First time seen"                    } },
    { "set":            { "if": "ctx.rule.level == 9",          "field": "rule.category", "value": "Error from invalid source"  } },
    { "set":            { "if": "ctx.rule.level == 10",                 "field": "rule.category", "value": "Multiple user generated errors"     } },
    { "set":            { "if": "ctx.rule.level == 11",                 "field": "rule.category", "value": "Integrity checking warning"         } },
    { "set":            { "if": "ctx.rule.level == 12",                 "field": "rule.category", "value": "High importance event"              } },
    { "set":            { "if": "ctx.rule.level == 13",                 "field": "rule.category", "value": "Unusal error (high importance)"     } },
    { "set":            { "if": "ctx.rule.level == 14",                 "field": "rule.category", "value": "High importance security event"     } },
    { "set":            { "if": "ctx.rule.level == 15",                 "field": "rule.category", "value": "Severe attack"                      } },
    { "set":            { "if": "ctx.rule.level <= 7",                  "field": "event.severity", "value": 1, "override": true }  },
    { "set":            { "if": "ctx.rule.level >= 8 && ctx.rule.level <= 11",     "field": "event.severity", "value": 2, "override": true }  },
    { "set":            { "if": "ctx.rule.level >= 12",          "field": "event.severity", "value": 3, "override": true }  },
    { "append":         { "if": "ctx.rule.level != null",               "field": "tags", "value": ["alert"]                                     } },
    { "remove": { "field": [ "predecoder", "decoder" ], "ignore_missing": true, "ignore_failure": false } },
    { "pipeline":       { "name": "common" } }
  ]
 }
@@ -1,25 +0,0 @@
 {
  "description" : "sguild_nids",
  "processors" : [
    { 
 	"dissect": {
 		"field": "message",
 		"pattern" : "%{} %{} %{}  Alert Received: %{} %{priority} %{classification} %{interface} {%{alerttime}} %{} %{} {%{alert}} %{source_ip} %{destination_ip} %{protocol} %{source_port} %{destination_port} %{gid} %{sid} %{rev} ",
 		"on_failure": [ { "drop" : { } } ]
 	}
    },
    { "set":		{ "if": "ctx.protocol == '1'",		"field": "protocol", "value": "ICMP"							} },
    { "set":		{ "if": "ctx.protocol == '6'",		"field": "protocol", "value": "TCP" 							} },
    { "set":		{ "if": "ctx.protocol == '17'",		"field": "protocol", "value": "UDP" 							} },
    { "remove":		{ "if": "ctx.source_ip == '{}'",  	"field": "source_ip"									} },
    { "remove":		{ "if": "ctx.destination_ip == '{}'", 	"field": "destination_ip"								} },
    { "remove":		{ "if": "ctx.protocol == '{}'", 	"field": "protocol"									} },
    { "remove":		{ "if": "ctx.source_port == '{}'",  	"field": "source_port"									} },
    { "remove":		{ "if": "ctx.destination_port == '{}'", "field": "destination_port"								} },
    { "set":		{ "field": "type", "value": "snort" 												} },
    { "rename":		{ "field": "@timestamp",	"target_field": "timestamp",						"ignore_missing": true  } },
    { "date":		{ "field": "alerttime",		"target_field": "@timestamp",	"formats": ["yyyy-MM-dd HH:mm:ss"],	"ignore_failure": true  } },
    { "remove":		{ "field": "alerttime",											"ignore_missing": true  } },
    { "pipeline": 	{ "name": "common_nids" 													} }
  ]
 }
@@ -1,21 +0,0 @@
 {
  "description" : "snort",
  "processors" : [
    { 
 	"dissect": {
 		"field": "message",
 		"pattern" : "[%{gid}:%{sid}:%{rev}] %{alert} [Classification: %{classification}] [Priority: %{priority}]: <%{interface}> {%{protocol}} %{source_ip_port} -> %{destination_ip_port}",
 		"on_failure": [ { "drop" : { } } ]
 	}
    },
    { "split": { "field": "source_ip_port", 		"separator": ":", 			"ignore_failure": true 	} },
    { "split": { "field": "destination_ip_port", 	"separator": ":", 			"ignore_failure": true 	} },
    { "rename":{ "field": "source_ip_port.1",		"target_field": "source_port",		"ignore_failure": true 	} },
    { "rename":{ "field": "destination_ip_port.1",	"target_field": "destination_port",	"ignore_failure": true 	} },
    { "rename":{ "field": "source_ip_port.0",		"target_field": "source_ip",		"ignore_failure": true 	} },
    { "rename":{ "field": "destination_ip_port.0",	"target_field": "destination_ip",	"ignore_failure": true 	} },
    { "remove":{ "field": "source_ip_port", 							"ignore_failure": true	} },
    { "remove":{ "field": "destination_ip_port", 						"ignore_failure": true	} },
    { "pipeline": { "name": "common_nids" } }
  ]
 }
@@ -6,7 +6,8 @@
    { "rename":         { "field": "message2.scan",        "target_field": "scan",          "ignore_missing": true  } },
    { "rename":         { "field": "message2.request",        "target_field": "request",          "ignore_missing": true  } },
    { "rename":         { "field": "scan.hash",        "target_field": "hash",          "ignore_missing": true  } },
-    { "grok":           { "field": "request.attributes.filename",       "patterns": ["-%{WORD:log.id.fuid}-"] } },
+
    { "grok":           { "if": "ctx.request?.attributes?.filename != null",   "field": "request.attributes.filename",       "patterns": ["-%{WORD:log.id.fuid}-"], "ignore_failure": true } },
    { "foreach":
      {
        "if": "ctx.scan?.exiftool?.keys !=null",
@@ -19,8 +20,29 @@
          }
      }
    },
    { "foreach":
      {
        "if": "ctx.scan?.yara?.meta !=null",
          "field": "scan.yara.meta",
          "processor":{
            "set": {
              "field": "rule.{{_ingest._value.identifier}}",
              "value": "{{_ingest._value.value}}"
            }
          }
      }
    },
    { "set":         { "if": "ctx.scan?.yara?.matches != null", "field": "rule.name",        "value": "{{scan.yara.matches.0}}" }},
    { "set":         { "if": "ctx.scan?.yara?.matches != null", "field": "dataset",        "value": "alert", "override": true }},
    { "rename":         { "field": "file.flavors.mime", "target_field": "file.mime_type",        "ignore_missing": true }},
    { "set":         { "if": "ctx.rule?.name != null && ctx.rule?.score == null",   "field": "event.severity", "value": 3, "override": true }  },
    { "convert" :    { "if": "ctx.rule?.score != null", "field" : "rule.score","type": "integer"}},
    { "set":         { "if": "ctx.rule?.score != null && ctx.rule?.score >= 0 && ctx.rule?.score <= 49",   "field": "event.severity", "value": 1, "override": true }  },
    { "set":         { "if": "ctx.rule?.score != null && ctx.rule?.score >= 50 && ctx.rule?.score <=69",   "field": "event.severity", "value": 2, "override": true }  },
    { "set":         { "if": "ctx.rule?.score != null && ctx.rule?.score >= 70 && ctx.rule?.score <=89",   "field": "event.severity", "value": 3, "override": true }  },
    { "set":         { "if": "ctx.rule?.score != null && ctx.rule?.score >= 90",   "field": "event.severity", "value": 4, "override": true }  },
    { "set":         { "field": "observer.name",        "value": "{{agent.name}}" }},
-    { "remove":         { "field": ["host", "path", "message", "scan.exiftool.keys"],                                         "ignore_missing": true  } },
+    { "remove":         { "field": ["host", "path", "message", "scan.exiftool.keys", "scan.yara.meta"],                                         "ignore_missing": true  } },
    { "pipeline":       { "name": "common"                                                                                   } }
  ]
 }
@@ -7,9 +7,6 @@
    { "rename":{ "field": "rule.signature_id",     "target_field": "rule.uuid", "ignore_failure": true  } },
    { "rename":{ "field": "rule.signature_id",     "target_field": "rule.signature", "ignore_failure": true  } },
    { "rename":{ "field": "message2.payload_printable",     "target_field": "network.data.decoded", "ignore_failure": true  } },
-    { "set":         { "if": "ctx.rule.severity == 3",   "field": "event.severity", "value": 1, "override": true }  },
+    { "pipeline": { "name": "common.nids" } }
    { "set":         { "if": "ctx.rule.severity == 2",   "field": "event.severity", "value": 2, "override": true }  },
    { "set":         { "if": "ctx.rule.severity == 1",   "field": "event.severity", "value": 3, "override": true }  },
    { "pipeline": { "name": "common" } }
  ]
 }
@@ -12,6 +12,9 @@
                "ignore_failure": true
        }
    },
    { "grok":           { "field": "message",       "patterns": ["<%{INT:syslog.priority}>%{DATA:syslog.timestamp} %{WORD:source.application}: %{GREEDYDATA:real_message}"], "ignore_failure": false } },
    { "set": { "if": "ctx.source.application == 'filterlog'", "field": "dataset", "value": "firewall" } },
    { "pipeline": { "if": "ctx.dataset == 'firewall'", "name": "filterlog" } },
    { "pipeline":       { "name": "common"                                             } }
  ]
 }
@@ -30,25 +30,40 @@
    { "rename": 	   { "field": "winlog.event_data.DestinationHostname", 	  "target_field": "destination.hostname",	"ignore_missing": true 	} },
    { "rename": 	   { "field": "winlog.event_data.DestinationIp", 		      "target_field": "destination.ip",	"ignore_missing": true 	} },
    { "rename": 	   { "field": "winlog.event_data.DestinationPort", 	      "target_field": "destination.port",	"ignore_missing": true 	} },
    { "rename":            { "field": "winlog.event_data.image",                                    "target_field": "process.executable",               "ignore_missing": true  } }, 
    { "rename": 	   { "field": "winlog.event_data.Image", 			            "target_field": "process.executable",		"ignore_missing": true 	} },
    { "rename":            { "field": "winlog.event_data.processID",                            "target_field": "process.pid",          "ignore_missing": true  } }, 
    { "rename": 	   { "field": "winlog.event_data.ProcessID", 			        "target_field": "process.pid",		"ignore_missing": true 	} },
    { "rename":            { "field": "winlog.event_data.processGuid",                        "target_field": "process.entity_id",              "ignore_missing": true  } },
    { "rename": 	   { "field": "winlog.event_data.ProcessGuid", 			      "target_field": "process.entity_id",		"ignore_missing": true 	} },
    { "rename":            { "field": "winlog.event_data.commandLine",                        "target_field": "process.command_line",           "ignore_missing": true  } },    
    { "rename": 	   { "field": "winlog.event_data.CommandLine", 			      "target_field": "process.command_line",		"ignore_missing": true 	} },
    { "rename":            { "field": "winlog.event_data.currentDirectory",                     "target_field": "process.working_directory",            "ignore_missing": true  } },
    { "rename": 	   { "field": "winlog.event_data.CurrentDirectory", 			"target_field": "process.working_directory",		"ignore_missing": true 	} },
    { "rename":            { "field": "winlog.event_data.description",                        "target_field": "process.pe.description",         "ignore_missing": true  } },
    { "rename": 	   { "field": "winlog.event_data.Description", 			      "target_field": "process.pe.description",		"ignore_missing": true 	} },
    { "rename":            { "field": "winlog.event_data.product",                                "target_field": "process.pe.product",         "ignore_missing": true  } },
    { "rename": 	   { "field": "winlog.event_data.Product", 			          "target_field": "process.pe.product",		"ignore_missing": true 	} },
    { "rename":            { "field": "winlog.event_data.company",                                "target_field": "process.pe.company",         "ignore_missing": true  } }, 
    { "rename": 	   { "field": "winlog.event_data.Company", 			          "target_field": "process.pe.company",		"ignore_missing": true 	} },
    { "rename":            { "field": "winlog.event_data.originalFileName",                     "target_field": "process.pe.original_file_name",                "ignore_missing": true  } }, 
    { "rename": 	   { "field": "winlog.event_data.OriginalFileName", 			"target_field": "process.pe.original_file_name",		"ignore_missing": true 	} },
    { "rename":            { "field": "winlog.event_data.fileVersion",                        "target_field": "process.pe.file_version",                "ignore_missing": true  } },
    { "rename": 	   { "field": "winlog.event_data.FileVersion", 			      "target_field": "process.pe.file_version",		"ignore_missing": true 	} },
    { "rename":            { "field": "winlog.event_data.parentCommandLine",                    "target_field": "process.parent.command_line",          "ignore_missing": true  } },
    { "rename": 	   { "field": "winlog.event_data.ParentCommandLine", 			"target_field": "process.parent.command_line",		"ignore_missing": true 	} },
    { "rename":            { "field": "winlog.event_data.parentImage",                        "target_field": "process.parent.executable",              "ignore_missing": true  } },
    { "rename": 	   { "field": "winlog.event_data.ParentImage", 			      "target_field": "process.parent.executable",		"ignore_missing": true 	} },
    { "rename":            { "field": "winlog.event_data.parentProcessGuid",                    "target_field": "process.parent.entity_id",             "ignore_missing": true  } }, 
    { "rename": 	   { "field": "winlog.event_data.ParentProcessGuid", 			"target_field": "process.parent.entity_id",		"ignore_missing": true 	} },
    { "rename":            { "field": "winlog.event_data.parentProcessId",                        "target_field": "process.ppid",               "ignore_missing": true  } },
    { "rename": 	   { "field": "winlog.event_data.ParentProcessId", 			  "target_field": "process.ppid",		"ignore_missing": true 	} },
    { "rename": 	   { "field": "winlog.event_data.Protocol", 	              "target_field": "network.transport",	"ignore_missing": true 	} },
    { "rename": 	   { "field": "winlog.event_data.User", 			            "target_field": "user.name",		"ignore_missing": true 	} },
    { "rename": 	   { "field": "winlog.event_data.SourceHostname", 	      "target_field": "source.hostname",	"ignore_missing": true 	} },
    { "rename": 	   { "field": "winlog.event_data.SourceIp", 		          "target_field": "source.ip",		"ignore_missing": true 	} },
    { "rename": 	   { "field": "winlog.event_data.SourcePort", 		        "target_field": "source.port",		"ignore_missing": true 	} },   
-    { "rename": 	   { "field": "winlog.event_data.targetFilename",		      "target_field": "file.target",	"ignore_missing": true 	} }
+    { "rename": 	   { "field": "winlog.event_data.targetFilename",		      "target_field": "file.target",	"ignore_missing": true 	} },
    { "rename":            { "field": "winlog.event_data.TargetFilename",                     "target_field": "file.target",    "ignore_missing": true  } }
  ]
-}
+}
@@ -1,7 +1,8 @@
 {
  "description" : "zeek.common",
  "processors" : [
-    { "rename":		{ "field": "@timestamp",	"target_field": "ingest.timestamp",	"ignore_missing": true					} },
+    { "rename":         { "if": "ctx.message2?.ts != null", "field": "@timestamp",      "target_field": "ingest.timestamp",     "ignore_missing": true                                  } },
    { "set":            { "if": "ctx.message2?.ts == null", "field": "ingest.timestamp",      "value": "{{ @timestamp }}"       } },
    { "rename":         { "field": "message2.uid",              "target_field": "log.id.uid",                  "ignore_missing": true  } },
    { "dot_expander":   { "field": "id.orig_h",                 "path": "message2",                     "ignore_failure": true  } },
    { "dot_expander":   { "field": "id.orig_p",                 "path": "message2",                     "ignore_failure": true  } },
@@ -12,9 +13,9 @@
    { "rename":         { "field": "message2.id.orig_p",        "target_field": "source.port",          "ignore_missing": true  } },
    { "rename":         { "field": "message2.id.resp_h",        "target_field": "destination.ip",       "ignore_missing": true  } },
    { "rename":         { "field": "message2.id.resp_p",        "target_field": "destination.port",     "ignore_missing": true  } },
-    { "set":         { "field": "client.ip",        "value": "{{source.ip}}"           } },
+    { "set":         { "if": "ctx.source?.ip != null", "field": "client.ip",        "value": "{{source.ip}}"           } },
    { "set":         { "if": "ctx.source?.port != null", "field": "client.port",        "value": "{{source.port}}"       } },
-    { "set":         { "field": "server.ip",        "value": "{{destination.ip}}"  } },
+    { "set":         { "if": "ctx.destination?.ip != null", "field": "server.ip",        "value": "{{destination.ip}}"  } },
    { "set":         { "if": "ctx.destination?.port != null", "field": "server.port",        "value": "{{destination.port}}"  } },
    { "set":         { "field": "observer.name",        "value": "{{agent.name}}"  } },
    { "date":		{ "field": "message2.ts",	"target_field": "@timestamp",	"formats": ["ISO8601", "UNIX"], "ignore_failure": true	} },
@@ -3,15 +3,6 @@
  "processors" : [
    { "remove":         { "field": ["host"],     "ignore_failure": true                                                                  } },
    { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
    { "rename": 	{ "field": "message2.uid", 	 	"target_field": "log.id.uid",			"ignore_missing": true 	} },
    { "dot_expander": 	{ "field": "id.orig_h", 		"path": "message2", 			"ignore_failure": true 	} },
    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source.ip",		"ignore_missing": true 	} },
    { "dot_expander": 	{ "field": "id.orig_p", 		"path": "message2",			"ignore_failure": true 	} },
    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source.port",		"ignore_missing": true 	} },
    { "dot_expander": 	{ "field": "id.resp_h", 		"path": "message2",			"ignore_failure": true 	} },
    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination.ip",	"ignore_missing": true 	} },
    { "dot_expander": 	{ "field": "id.resp_p", 		"path": "message2",			"ignore_failure": true 	} },
    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination.port",	"ignore_missing": true 	} },
    { "rename": 	{ "field": "message2.tunnel_type", 	"target_field": "tunnel.type",		"ignore_missing": true 	} },
    { "rename": 	{ "field": "message2.action", 		"target_field": "event.action",		"ignore_missing": true 	} },
    { "pipeline":       { "name": "zeek.common"                                                                                   } }
@@ -27,7 +27,11 @@ echo -n "Waiting for ElasticSearch..."
 COUNT=0
 ELASTICSEARCH_CONNECTED="no"
 while [[ "$COUNT" -le 240 ]]; do
    {% if grains['role'] in ['so-node','so-heavynode'] %}
 	curl ${ELASTICSEARCH_AUTH} -k --output /dev/null --silent --head --fail https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"
 	{% else %}
 	curl ${ELASTICSEARCH_AUTH} --output /dev/null --silent --head --fail http://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"
 	{% endif %}
 	if [ $? -eq 0 ]; then
 		ELASTICSEARCH_CONNECTED="yes"
 		echo "connected!"
@@ -47,7 +51,11 @@ fi
 cd ${ELASTICSEARCH_INGEST_PIPELINES}
 echo "Loading pipelines..."
 {% if grains['role'] in ['so-node','so-heavynode'] %}
 for i in *; do echo $i; RESPONSE=$(curl ${ELASTICSEARCH_AUTH} -k -XPUT https://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_ingest/pipeline/$i -H 'Content-Type: application/json' -d@$i 2>/dev/null); echo $RESPONSE; if [[ "$RESPONSE" == *"error"* ]]; then RETURN_CODE=1; fi; done
 {% else %}
 for i in *; do echo $i; RESPONSE=$(curl ${ELASTICSEARCH_AUTH} -XPUT http://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_ingest/pipeline/$i -H 'Content-Type: application/json' -d@$i 2>/dev/null); echo $RESPONSE; if [[ "$RESPONSE" == *"error"* ]]; then RETURN_CODE=1; fi; done
 {% endif %}
 echo
 cd - >/dev/null
@@ -8,5 +8,10 @@ protocols:
 - TLSv1.2
 ciphers:
 - TLS_RSA_WITH_AES_128_CBC_SHA256
 - TLS_RSA_WITH_AES_256_GCM_SHA384
 transport.encrypted: true
 {%- if grains['role'] in ['so-node','so-heavynode'] %}
 http.encrypted: true
 {%- else %}
 http.encrypted: false
 {%- endif %}
@@ -150,6 +150,7 @@ sotls:
    - source: salt://elasticsearch/files/sotls.yml
    - user: 930
    - group: 939
    - template: jinja
 #sync templates to /opt/so/conf/elasticsearch/templates
 {% for TEMPLATE in TEMPLATES %}
@@ -228,6 +229,7 @@ so-elasticsearch-pipelines-file:
    - user: 930
    - group: 939
    - mode: 754
    - template: jinja
 so-elasticsearch-pipelines:
 cmd.run:
@@ -237,11 +239,12 @@ so-elasticsearch-pipelines:
      - file: esyml
      - file: so-elasticsearch-pipelines-file
-{% if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode', 'so-import'] and TEMPLATES %}
+{% if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-node', 'so-import'] and TEMPLATES %}
 so-elasticsearch-templates:
  cmd.run:
-    - name: /usr/sbin/so-elasticsearch-templates
+    - name: /usr/sbin/so-elasticsearch-templates-load
    - cwd: /opt/so
    - template: jinja
 {% endif %}
 {% else %}
@@ -7,6 +7,7 @@
    "number_of_shards":1,
    "index.refresh_interval":"30s",
    "index.routing.allocation.require.box_type":"hot",
    "index.mapping.total_fields.limit": "1500",
    "analysis": {
      "analyzer": {
        "es_security_analyzer": {
@@ -41,7 +42,25 @@
       "dynamic":false,
       "date_detection":false,
       "dynamic_templates": [
-        {
+       {
          "ip_address": {
            "match_mapping_type": "string",
            "path_match": "*.ip",
            "mapping": {
              "type": "ip"
            }
          }
       },
       {
          "port": {
            "match_mapping_type": "string",
            "path_match": "*.port",
            "mapping": {
              "type": "integer"
            }
          }
       },
       {
          "strings": {
            "match_mapping_type": "string",
            "mapping": {
@@ -52,15 +71,14 @@
               },
               "security": {
                 "type": "text",
-                 "analyzer": "es_security_analyzer", 
+                 "analyzer": "es_security_analyzer",
-                 "search_analyzer": "es_security_search_analyzer", 
+                 "search_analyzer": "es_security_search_analyzer",
-                 "search_quote_analyzer": "es_security_search_quote_analyzer" 
+                 "search_quote_analyzer": "es_security_search_quote_analyzer"
               }
             }
            }
          }
-        }
+       }],
      ],
       "properties":{
        "@timestamp":{
          "type":"date"
@@ -235,11 +253,29 @@
          "type":"object",
          "dynamic": true
        },
        "intel":{
          "type":"object",
          "dynamic": true,
          "properties":{
            "indicator":{
              "type":"text",
              "fields":{
                "keyword":{
                  "type":"keyword"
                }
              }
            }
          } 
        },
        "interface":{
          "type":"object",
          "dynamic": true
        },
-	      "irc":{
+	"ip":{
          "type":"object",
          "dynamic": true
        }, 
 	"irc":{
          "type":"object",
          "dynamic": true
        },
@@ -255,7 +291,7 @@
          "type":"object",
          "dynamic": true
        },
-	      "message":{  
+	"message":{  
          "type":"text",
          "fields":{  
            "keyword":{  
@@ -333,7 +369,15 @@
        },
        "rule":{
          "type":"object",
-          "dynamic": true
+          "dynamic":true,
          "properties":{
            "score":{
              "type":"long"
            },
            "uuid":{
              "type":"keyword"
            }
          }
        },
 	      "scan":{
          "type":"object",
@@ -444,6 +488,9 @@
            },
            "event_data":{
              "type":"object"
            },
            "version":{
              "type":"long"
            }
        }
      },
@@ -82,7 +82,7 @@ filebeat.inputs:
    module: syslog
    dataset: syslog
  pipeline: "syslog"
-  index: "so-syslog-%{+yyyy.MM.dd}"
+  index: "so-syslog"
  processors:
  - drop_fields:
      fields: ["source", "prospector", "input", "offset", "beat"]
@@ -95,7 +95,7 @@ filebeat.inputs:
    module: syslog
    dataset: syslog
  pipeline: "syslog"
-  index: "so-syslog-%{+yyyy.MM.dd}"
+  index: "so-syslog"
  processors:
  - drop_fields:
      fields: ["source", "prospector", "input", "offset", "beat"]
@@ -203,15 +203,14 @@ filebeat.inputs:
 - type: log
  paths:
-    - /wazuh/alerts/alerts.json
+    - /wazuh/archives/archives.json
  fields:
    module: ossec
    dataset: alert
    category: host
  processors:
  - drop_fields:
      fields: ["source", "prospector", "input", "offset", "beat"]
-
+  pipeline: "ossec"
  fields_under_root: true
  clean_removed: false
  close_removed: false
@@ -260,22 +259,22 @@ output.elasticsearch:
  pipelines:
    - pipeline: "%{[module]}.%{[dataset]}"
  indices:
-    - index: "so-import-%{+yyyy.MM.dd}"
+    - index: "so-import"
      when.contains:
        tags: "import"
-    - index: "so-zeek-%{+yyyy.MM.dd}"
+    - index: "so-zeek"
      when.contains:
        module: "zeek"
-    - index: "so-ids-%{+yyyy.MM.dd}"
+    - index: "so-ids"
      when.contains:
        module: "suricata"
-    - index: "so-ossec-%{+yyyy.MM.dd}"
+    - index: "so-ossec"
      when.contains:
        module: "ossec"
-    - index: "so-osquery-%{+yyyy.MM.dd}"
+    - index: "so-osquery"
      when.contains:
        module: "osquery"
-    - index: "so-strelka-%{+yyyy.MM.dd}"
+    - index: "so-strelka"
      when.contains:
        module: "strelka"
@@ -18,6 +18,9 @@
 {% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %}
 {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %}
 {% set LOCALHOSTNAME = salt['grains.get']('host') %}
 {% set MAININT = salt['pillar.get']('host:mainint') %}
 {% set LOCALHOSTIP = salt['grains.get']('ip_interfaces').get(MAININT)[0] %}
 {% set MANAGER = salt['grains.get']('master') %}
 {% set MANAGERIP = salt['pillar.get']('global:managerip', '') %}
 {% set FEATURES = salt['pillar.get']('elastic:features', False) %}
@@ -44,6 +47,12 @@ filebeatpkidir:
    - user: 939
    - group: 939
    - makedirs: True
 fileregistrydir:
  file.directory:
    - name: /opt/so/conf/filebeat/registry
    - user: 939
    - group: 939
    - makedirs: True
 # This needs to be owned by root
 filebeatconfsync:
  file.managed:
@@ -60,7 +69,7 @@ so-filebeat:
    - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-filebeat:{{ VERSION }}{{ FEATURES }}
    - hostname: so-filebeat
    - user: root
-    - extra_hosts: {{ MANAGER }}:{{ MANAGERIP }}
+    - extra_hosts: {{ MANAGER }}:{{ MANAGERIP }},{{ LOCALHOSTNAME }}:{{ LOCALHOSTIP }}
    - binds:
      - /nsm:/nsm:ro
      - /opt/so/log/filebeat:/usr/share/filebeat/logs:rw
@@ -69,6 +78,7 @@ so-filebeat:
      - /nsm/wazuh/logs/archives:/wazuh/archives:ro
      - /opt/so/conf/filebeat/etc/pki/filebeat.crt:/usr/share/filebeat/filebeat.crt:ro
      - /opt/so/conf/filebeat/etc/pki/filebeat.key:/usr/share/filebeat/filebeat.key:ro
      - /opt/so/conf/filebeat/registry:/usr/share/filebeat/data/registry:rw
      - /etc/ssl/certs/intca.crt:/usr/share/filebeat/intraca.crt:ro
    - port_bindings:
        - 0.0.0.0:514:514/udp
@@ -81,4 +91,4 @@ filebeat_state_not_allowed:
  test.fail_without_changes:
    - name: filebeat_state_not_allowed
-{% endif %}
+{% endif %}
@@ -416,6 +416,7 @@ role:
          manager:
            portgroups:
              - {{ portgroups.elasticsearch_node }}
              - {{ portgroups.elasticsearch_rest }}
          dockernet:
            portgroups:
              - {{ portgroups.elasticsearch_node }}
@@ -454,6 +455,7 @@ role:
          manager:
            portgroups:
              - {{ portgroups.elasticsearch_node }}
              - {{ portgroups.elasticsearch_rest }}
          dockernet:
            portgroups:
              - {{ portgroups.elasticsearch_node }}
@@ -4137,7 +4137,7 @@
    ]
  },
  "time": {
-    "from": "now-15m",
+    "from": "now-1h",
    "to": "now"
  },
  "timepicker": {
@@ -4168,5 +4168,5 @@
  "timezone": "browser",
  "title": "Manager Node - {{ SERVERNAME }} Overview",
  "uid": "{{ UID }}",
-  "version": 3
+  "version": 1
 }
@@ -4883,7 +4883,7 @@
      ]
    },
    "time": {
-      "from": "now-30m",
+      "from": "now-1h",
      "to": "now"
    },
    "timepicker": {
@@ -4912,7 +4912,7 @@
      ]
    },
    "timezone": "browser",
-    "title": "Search Node - {{ SERVERNAME }} Overview",
+    "title": "ManagerSearch Node - {{ SERVERNAME }} Overview",
    "uid": "{{ UID }}",
    "variables": {
      "list": []
@@ -3658,7 +3658,7 @@
    ]
  },
  "time": {
-    "from": "now-30m",
+    "from": "now-1h",
    "to": "now"
  },
  "timepicker": {
@@ -4231,6 +4231,139 @@
        "alignLevel": null
      }
    },
    {
      "aliasColors": {},
      "bars": true,
      "dashLength": 10,
      "dashes": false,
      "datasource": "InfluxDB",
      "fieldConfig": {
        "defaults": {
          "custom": {}
        },
        "overrides": []
      },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 5,
        "w": 4,
        "x": 16,
        "y": 25
      },
      "hiddenSeries": false,
      "id": 71,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": false,
        "total": false,
        "values": false
      },
      "lines": false,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": {
        "dataLinks": []
      },
      "percentage": false,
      "pointradius": 2,
      "points": true,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "groupBy": [
            {
              "params": [
                "$__interval"
              ],
              "type": "time"
            },
            {
              "params": [
                "null"
              ],
              "type": "fill"
            }
          ],
          "measurement": "zeekcaptureloss",
          "orderByTime": "ASC",
          "policy": "autogen",
          "refId": "A",
          "resultFormat": "time_series",
          "select": [
            [
              {
                "params": [
                  "loss"
                ],
                "type": "field"
              },
              {
                "params": [],
                "type": "mean"
              }
            ]
          ],
          "tags": [
            {
              "key": "host",
              "operator": "=",
              "value": "{{ SERVERNAME }}"
            }
          ]
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "{{ SERVERNAME }} - Zeek Capture Loss",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "$$hashKey": "object:198",
          "decimals": 1,
          "format": "percent",
          "label": "",
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        },
        {
          "$$hashKey": "object:199",
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": false
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    },    
    {
      "aliasColors": {},
      "bars": false,
@@ -4428,7 +4561,7 @@
              "type": "fill"
            }
          ],
-          "measurement": "brodrop",
+          "measurement": "zeekdrop",
          "orderByTime": "ASC",
          "policy": "default",
          "refId": "A",
@@ -4867,8 +5000,8 @@
      "fillGradient": 0,
      "gridPos": {
        "h": 5,
-        "w": 8,
+        "w": 4,
-        "x": 16,
+        "x": 20,
        "y": 30
      },
      "hiddenSeries": false,
@@ -6518,7 +6651,7 @@
    ]
  },
  "time": {
-    "from": "now-15m",
+    "from": "now-1h",
    "to": "now"
  },
  "timepicker": {
@@ -6548,6 +6681,6 @@
  },
  "timezone": "browser",
  "title": "Standalone Mode - {{ SERVERNAME }} Overview",
-  "uid": "so_overview",
+  "uid": "{{ UID }}",
-  "version": 4
+  "version": 1
 }
@@ -7,6 +7,7 @@
 {% set MANAGER = salt['grains.get']('master') %}
 {% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %}
 {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %}
 {% set ADMINPASS = salt['pillar.get']('secrets:grafana_admin') %}
 {% if grains['role'] in ['so-manager', 'so-managersearch', 'so-eval', 'so-standalone'] and GRAFANA == 1 %}
@@ -229,7 +230,7 @@ so-grafana:
      - /opt/so/conf/grafana/etc/dashboards:/etc/grafana/provisioning/dashboards:rw
      - /opt/so/conf/grafana/grafana_dashboards:/etc/grafana/grafana_dashboards:rw
    - environment:
-      - GF_SECURITY_ADMIN_PASSWORD=augusta
+      - GF_SECURITY_ADMIN_PASSWORD={{ ADMINPASS }}
    - port_bindings:
      - 0.0.0.0:3000:3000
    - watch:
@@ -17,16 +17,16 @@
 --disable=/opt/so/idstools/etc/disable.conf
 --enable=/opt/so/idstools/etc/enable.conf
 --modify=/opt/so/idstools/etc/modify.conf
-  {%- if RULESET == 'ETOPEN' -%}
+  {%- if RULESET == 'ETOPEN' %}
 --etopen
-  {%- elif RULESET == 'ETPRO' -%}
+  {%- elif RULESET == 'ETPRO' %}
 --etpro={{ OINKCODE }}
-  {%- elif RULESET == 'TALOS' -%}
+  {%- elif RULESET == 'TALOS' %}
 --url=https://www.snort.org/rules/snortrules-snapshot-2983.tar.gz?oinkcode={{ OINKCODE }}
-  {%- endif -%}
+  {%- endif %}
-{%- endif -%}
+{%- endif %}
-{%- if URLS != None -%}
+{%- if URLS != None %}
-{%- for URL in URLS -%}
+{%- for URL in URLS %}
 --url={{ URL }}
-{%- endfor -%}
+{%- endfor %}
-{%- endif -%}
+{%- endif %}
@@ -16,6 +16,14 @@ influxconfdir:
    - name: /opt/so/conf/influxdb/etc
    - makedirs: True
 influxlogdir:
  file.directory:
    - name: /opt/so/log/influxdb
    - dir_mode: 775
    - user: 939
    - group: 939
    - makedirs: True
 influxdbdir:
  file.directory:
    - name: /nsm/influxdb
@@ -36,6 +44,7 @@ so-influxdb:
    - environment:
      - INFLUXDB_HTTP_LOG_ENABLED=false
    - binds:
      - /opt/so/log/influxdb/:/log:rw
      - /opt/so/conf/influxdb/etc/influxdb.conf:/etc/influxdb/influxdb.conf:ro
      - /nsm/influxdb:/var/lib/influxdb:rw
      - /etc/pki/influxdb.crt:/etc/ssl/influxdb.crt:ro
@@ -1,5 +1,6 @@
 {%- set MANAGER = salt['grains.get']('master') %}
 {%- set THREADS = salt['pillar.get']('logstash_settings:ls_input_threads', '') %}
 {% set BATCH = salt['pillar.get']('logstash_settings:ls_pipeline_batch_size', 125) %}
 input {
 	redis {
@@ -10,5 +11,6 @@ input {
 		key => 'logstash:unparsed'
 		type => 'redis-input'
 		threads => {{ THREADS }}
 		batch_count => {{ BATCH }}
 	}
 }
@@ -9,10 +9,14 @@ output {
    elasticsearch {
      pipeline => "%{module}.%{dataset}"
      hosts => "{{ ES }}"
-      index => "so-zeek-%{+YYYY.MM.dd}"
+      index => "so-zeek"
      template_name => "so-zeek"
      template => "/templates/so-zeek-template.json"
      template_overwrite => true
      {%- if grains['role'] in ['so-node','so-heavynode'] %}
      ssl => true
      ssl_certificate_verification => false
      {%- endif %}
    }
  }
 }
@@ -9,10 +9,14 @@ output {
    elasticsearch {
      pipeline => "%{module}.%{dataset}"
      hosts => "{{ ES }}"
-      index => "so-import-%{+YYYY.MM.dd}"
+      index => "so-import"
      template_name => "so-import"
      template => "/templates/so-import-template.json"
      template_overwrite => true
      {%- if grains['role'] in ['so-node','so-heavynode'] %}
      ssl => true
      ssl_certificate_verification => false
      {%- endif %}
    }
  }
 }
@@ -8,10 +8,14 @@ output {
  if [event_type] == "sflow" {
    elasticsearch {
      hosts => "{{ ES }}"
-      index => "so-flow-%{+YYYY.MM.dd}"
+      index => "so-flow"
      template_name => "so-flow"
      template => "/templates/so-flow-template.json"
      template_overwrite => true
      {%- if grains['role'] in ['so-node','so-heavynode'] %}
      ssl => true
      ssl_certificate_verification => false
      {%- endif %}
    }
  }
 }
@@ -8,10 +8,14 @@ output {
  if [event_type] == "ids" and "import" not in [tags] {
    elasticsearch {
      hosts => "{{ ES }}"
-      index => "so-ids-%{+YYYY.MM.dd}"
+      index => "so-ids"
      template_name => "so-ids"
      template => "/templates/so-ids-template.json"
      template_overwrite => true
      {%- if grains['role'] in ['so-node','so-heavynode'] %}
      ssl => true
      ssl_certificate_verification => false
      {%- endif %}
    }
  }
 }
@@ -9,10 +9,14 @@ output {
    elasticsearch {
      pipeline => "%{module}"
      hosts => "{{ ES }}"
-      index => "so-syslog-%{+YYYY.MM.dd}"
+      index => "so-syslog"
      template_name => "so-syslog"
      template => "/templates/so-syslog-template.json"
      template_overwrite => true
      {%- if grains['role'] in ['so-node','so-heavynode'] %}
      ssl => true
      ssl_certificate_verification => false
      {%- endif %}
    }
  }
 }
@@ -9,10 +9,14 @@ output {
    elasticsearch {
      pipeline => "%{module}.%{dataset}" 
      hosts => "{{ ES }}"
-      index => "so-osquery-%{+YYYY.MM.dd}"
+      index => "so-osquery"
      template_name => "so-osquery"
      template => "/templates/so-osquery-template.json"
      template_overwrite => true
      {%- if grains['role'] in ['so-node','so-heavynode'] %}
      ssl => true
      ssl_certificate_verification => false
      {%- endif %}
    }
  }
 }
@@ -5,13 +5,17 @@
 {%- endif %}
 {% set FEATURES = salt['pillar.get']('elastic:features', False) %}
 output {
-  if "firewall" in [tags] {
+  if [dataset] =~ "firewall" {
    elasticsearch {
      hosts => "{{ ES }}"
-      index => "so-firewall-%{+YYYY.MM.dd}"
+      index => "so-firewall"
      template_name => "so-firewall"
      template => "/templates/so-firewall-template.json"
      template_overwrite => true
      {%- if grains['role'] in ['so-node','so-heavynode'] %}
      ssl => true
      ssl_certificate_verification => false
      {%- endif %}
    }
  }
 }
@@ -9,9 +9,13 @@ output {
    elasticsearch {
      pipeline => "%{module}.%{dataset}"
      hosts => "{{ ES }}"
-      index => "so-ids-%{+YYYY.MM.dd}"
+      index => "so-ids"
      template_name => "so-ids" 
      template => "/templates/so-ids-template.json"
      {%- if grains['role'] in ['so-node','so-heavynode'] %}
      ssl => true
      ssl_certificate_verification => false
      {%- endif %}
    }
  }
 }
@@ -9,10 +9,14 @@ output {
    elasticsearch {
      pipeline => "beats.common" 
      hosts => "{{ ES }}"
-      index => "so-beats-%{+YYYY.MM.dd}"
+      index => "so-beats"
      template_name => "so-beats"
      template => "/templates/so-beats-template.json"
      template_overwrite => true
      {%- if grains['role'] in ['so-node','so-heavynode'] %}
      ssl => true
      ssl_certificate_verification => false
      {%- endif %}
    }
  }
 }
@@ -7,12 +7,16 @@
 output {
  if [module] =~ "ossec" {
    elasticsearch {
-      pipeline => "%{module}.%{dataset}"
+      pipeline => "%{module}"
      hosts => "{{ ES }}"
-      index => "so-ossec-%{+YYYY.MM.dd}"
+      index => "so-ossec"
      template_name => "so-ossec"
      template => "/templates/so-ossec-template.json"
      template_overwrite => true
      {%- if grains['role'] in ['so-node','so-heavynode'] %}
      ssl => true
      ssl_certificate_verification => false
      {%- endif %}
    }
  }
 }
@@ -9,10 +9,14 @@ output {
    elasticsearch {
      pipeline => "%{module}.%{dataset}" 
      hosts => "{{ ES }}"
-      index => "so-strelka-%{+YYYY.MM.dd}"
+      index => "so-strelka"
      template_name => "so-strelka"
      template => "/templates/so-strelka-template.json"
      template_overwrite => true
      {%- if grains['role'] in ['so-node','so-heavynode'] %}
      ssl => true
      ssl_certificate_verification => false
      {%- endif %}
    }
  }
 }
@@ -6,5 +6,7 @@ MINION=$1
  echo "Adding $1" 
  cp /tmp/$MINION/pillar/$MINION.sls $local_salt_dir/pillar/minions/
-  cp --parents /tmp/$MINION/schedules/* $local_salt_dir/salt/patch/os/schedules/
+  if [ "$(ls -A /tmp/$MINION/schedules/)" ]; then
    cp /tmp/$MINION/schedules/* $local_salt_dir/salt/patch/os/schedules/
  fi
  rm -rf /tmp/$MINION
@@ -0,0 +1,14 @@
 # This state will import the initial default playbook database. 
 # If there is an existing playbook database, it will be overwritten - no backups are made.
 include:
  - mysql
 salt://playbook/files/OLD_playbook_db_init.sh:
  cmd.script:
    - cwd: /root
    - template: jinja
 'sleep 5':
  cmd.run
@@ -0,0 +1,19 @@
 {% set MAINIP = salt['pillar.get']('global:managerip') %}
 # This state will create the SecOps Automation user within Playbook
 include:
  - playbook
 wait_for_playbook:
  cmd.run:
    - name: until nc -z {{ MAINIP }} 3200; do sleep 1; done
    - timeout: 30
    - onchanges:
      - cmd: create_user
 create_user:
  cmd.script:
    - source: salt://playbook/files/automation_user_create.sh
    - cwd: /root
    - template: jinja
@@ -0,0 +1,8 @@
 #!/bin/sh
 # {%- set MYSQLPASS = salt['pillar.get']('secrets:mysql', None) %}
 default_salt_dir=/opt/so/saltstack/default
 docker cp $default_salt_dir/salt/playbook/files/OLD_playbook_db_init.sql so-mysql:/tmp/playbook_db_init.sql
 docker exec so-mysql /bin/bash -c "/usr/bin/mysql -b  -uroot -p{{MYSQLPASS}}  < /tmp/playbook_db_init.sql"
@@ -0,0 +1,50 @@
 #!/bin/bash
 # {%- set admin_pass = salt['pillar.get']('secrets:playbook_admin', None) -%}
 # {%- set automation_pass = salt['pillar.get']('secrets:playbook_automation', None) %}
 local_salt_dir=/opt/so/saltstack/local
 try_count=6
 interval=10
 while [[ $try_count -le 6 ]]; do
    if docker top "so-playbook" &>/dev/null; then
        automation_group=6
        # Create user and retrieve api_key and user_id from response
        mapfile -t automation_res < <(
            curl -s --location --request POST 'http://127.0.0.1:3200/playbook/users.json' --user "admin:{{ admin_pass }}" --header 'Content-Type: application/json' --data '{
                "user" : {
                    "login" : "automation",
                    "password": "{{ automation_pass }}",
                    "firstname": "SecOps",
                    "lastname": "Automation",
                    "mail": "automation2@localhost.local"
                }
            }' | jq -r '.user.api_key, .user.id'
        )
        automation_api_key=${automation_res[0]}
        automation_user_id=${automation_res[1]}
        # Add user_id from newly created user to Automation group
        curl -s --location --request POST "http://127.0.0.1:3200/playbook/groups/${automation_group}/users.json" \
            --user "admin:{{ admin_pass }}" \
            --header 'Content-Type: application/json' \
            --data "{
                \"user_id\" : ${automation_user_id}
            }"
        # Search for the needed keys in the global pillar file, if missing then add them
        if (grep -Pzq 'playbook:\n  api_key:.*' $local_salt_dir/pillar/global.sls); then
            sed -e '1h;2,$H;$!d;g' -e "s/playbook:\n  api_key:.*/playbook:\n  api_key: ${automation_api_key}/m" -i $local_salt_dir/pillar/global.sls
        else
            {
                echo "playbook:"
                echo "  api_key: ${automation_api_key}" 
            } >> $local_salt_dir/pillar/global.sls
        fi
    fi
    ((try_count++))
    sleep "${interval}s"
 done
@@ -1,7 +1,16 @@
-{%- set MYSQLPASS = salt['pillar.get']('secrets:mysql', None) -%}
+#!/bin/bash
-#!/bin/sh
+# {%- set MYSQLPASS = salt['pillar.get']('secrets:mysql', None) -%}
 # {%- set admin_pass = salt['pillar.get']('secrets:playbook_admin', None) %}
 default_salt_dir=/opt/so/saltstack/default
 # Generate salt + hash for admin user
 admin_salt=$(tr -dc "a-zA-Z0-9" < /dev/urandom | fold -w 32 | head -n 1)
 admin_stage1_hash=$(echo -n '{{ admin_pass }}' | sha1sum | awk '{print $1}')
 admin_hash=$(echo -n "${admin_salt}${admin_stage1_hash}" | sha1sum | awk '{print $1}')
 sed -i "s/ADMIN_HASH/${admin_hash}/g" $default_salt_dir/salt/playbook/files/playbook_db_init.sql
 sed -i "s/ADMIN_SALT/${admin_salt}/g" $default_salt_dir/salt/playbook/files/playbook_db_init.sql
 # Copy file to destination + execute SQL
 docker cp $default_salt_dir/salt/playbook/files/playbook_db_init.sql so-mysql:/tmp/playbook_db_init.sql
-docker exec so-mysql /bin/bash -c "/usr/bin/mysql -b  -uroot -p{{MYSQLPASS}}  < /tmp/playbook_db_init.sql"
+docker exec so-mysql /bin/bash -c "/usr/bin/mysql -b  -uroot -p{{MYSQLPASS}}  < /tmp/playbook_db_init.sql"
@@ -606,7 +606,7 @@ CREATE TABLE `groups_users` (
 LOCK TABLES `groups_users` WRITE;
 /*!40000 ALTER TABLE `groups_users` DISABLE KEYS */;
-INSERT INTO `groups_users` VALUES (6,9),(7,1);
+INSERT INTO `groups_users` VALUES (7,1);
 /*!40000 ALTER TABLE `groups_users` ENABLE KEYS */;
 UNLOCK TABLES;
@@ -1380,7 +1380,19 @@ CREATE TABLE `tokens` (
 LOCK TABLES `tokens` WRITE;
 /*!40000 ALTER TABLE `tokens` DISABLE KEYS */;
-INSERT INTO `tokens` VALUES (3,1,'feeds','6e5575602e1227c188cd85ef6d12608bb8701193','2020-04-26 13:10:46','2020-04-26 13:10:46'),(4,1,'session','999412fa9badda7423c6c654d6364c32c20b3eac','2020-04-26 18:07:03','2020-04-26 18:12:02'),(5,1,'session','124ad4acbf87a942426350e7ad028c1d119c3851','2020-04-26 18:17:11','2020-04-26 18:19:24'),(9,1,'session','2890c663e0552f26ddb92acad6ab3b6d05b92915','2020-04-26 18:51:15','2020-04-26 18:51:15'),(19,1,'session','b7ffb106ea0b34650dd9c1770f74c2b0ffe166b2','2020-05-01 16:52:33','2020-05-01 18:02:30'),(20,1,'session','f44cfcf918eef59ffda47991c431d9c2b2ac6113','2020-05-01 18:05:56','2020-05-01 18:05:56'),(23,9,'feeds','211918c9d7168979b5dc19bebb14573b928a5067','2020-05-01 18:26:17','2020-05-01 18:26:17'),(25,9,'api','de6639318502476f2fa5aa06f43f51fb389a3d7f','2020-05-01 18:26:31','2020-05-01 18:26:31'),(46,1,'session','2d0c8f8ae641c06d8c2362746846440d465d53c0','2020-05-06 20:48:01','2020-05-06 20:48:07'),(59,1,'session','2afe6590653d59a697d1436729c64f322a2eff82','2020-07-01 18:11:07','2020-07-01 20:30:43'),(61,1,'session','b01f95709ca1ab086a049cf9c5afd81ca9d4526e','2020-07-15 16:30:42','2020-07-15 16:31:40'),(62,1,'session','d29acdcd0b8e4ebf78ef8f696d3e76df7e2ab2ac','2020-08-17 14:51:59','2020-08-17 14:53:22');
+INSERT INTO `tokens` 
  VALUES 
    (3,1,'feeds','6e5575602e1227c188cd85ef6d12608bb8701193','2020-04-26 13:10:46','2020-04-26 13:10:46'),
    (4,1,'session','999412fa9badda7423c6c654d6364c32c20b3eac','2020-04-26 18:07:03','2020-04-26 18:12:02'),
    (5,1,'session','124ad4acbf87a942426350e7ad028c1d119c3851','2020-04-26 18:17:11','2020-04-26 18:19:24'),
    (9,1,'session','2890c663e0552f26ddb92acad6ab3b6d05b92915','2020-04-26 18:51:15','2020-04-26 18:51:15'),
    (19,1,'session','b7ffb106ea0b34650dd9c1770f74c2b0ffe166b2','2020-05-01 16:52:33','2020-05-01 18:02:30'),
    (20,1,'session','f44cfcf918eef59ffda47991c431d9c2b2ac6113','2020-05-01 18:05:56','2020-05-01 18:05:56'),
    (23,9,'feeds','211918c9d7168979b5dc19bebb14573b928a5067','2020-05-01 18:26:17','2020-05-01 18:26:17'),
    (46,1,'session','2d0c8f8ae641c06d8c2362746846440d465d53c0','2020-05-06 20:48:01','2020-05-06 20:48:07'),
    (59,1,'session','2afe6590653d59a697d1436729c64f322a2eff82','2020-07-01 18:11:07','2020-07-01 20:30:43'),
    (61,1,'session','b01f95709ca1ab086a049cf9c5afd81ca9d4526e','2020-07-15 16:30:42','2020-07-15 16:31:40'),
    (62,1,'session','d29acdcd0b8e4ebf78ef8f696d3e76df7e2ab2ac','2020-08-17 14:51:59','2020-08-17 14:53:22');
 /*!40000 ALTER TABLE `tokens` ENABLE KEYS */;
 UNLOCK TABLES;
@@ -1481,7 +1493,16 @@ CREATE TABLE `users` (
 LOCK TABLES `users` WRITE;
 /*!40000 ALTER TABLE `users` DISABLE KEYS */;
-INSERT INTO `users` VALUES (1,'admin','95535e9f7a386c412f20134ebb869c00cf346477','Admin','Admin',1,1,'2020-08-17 18:03:20','',NULL,'2020-04-26 13:08:34','2020-04-26 13:10:45','User',NULL,'all','5ceb2c95ce1593d4ba034d385ceefb2f',0,'2020-04-26 13:10:27'),(2,'','','','Anonymous users',0,1,NULL,'',NULL,'2020-04-26 13:08:38','2020-04-26 13:08:38','GroupAnonymous',NULL,'',NULL,0,NULL),(3,'','','','Non member users',0,1,NULL,'',NULL,'2020-04-26 13:08:38','2020-04-26 13:08:38','GroupNonMember',NULL,'',NULL,0,NULL),(4,'','','','Anonymous',0,0,NULL,'',NULL,'2020-04-26 13:09:44','2020-04-26 13:09:44','AnonymousUser',NULL,'only_my_events',NULL,0,NULL),(5,'','','','Analysts',0,1,NULL,'',NULL,'2020-04-26 18:43:40','2020-04-26 18:43:40','Group',NULL,'',NULL,0,NULL),(6,'','','','Automation',0,1,NULL,'',NULL,'2020-04-26 18:43:47','2020-04-26 18:43:47','Group',NULL,'',NULL,0,NULL),(7,'','','','Admins',0,1,NULL,'',NULL,'2020-04-26 18:43:58','2020-04-26 18:43:58','Group',NULL,'',NULL,0,NULL),(9,'automation','d2e7d78af1f0c0637765ae8cf1a359c4a30034c9','SecOps','Automation',0,1,'2020-05-01 18:26:17','en',NULL,'2020-04-26 18:47:46','2020-05-01 18:26:10','User',NULL,'none','41043e596f70e327e34fc99c861f5b31',0,'2020-05-01 18:26:10');
+INSERT INTO `users` 
  VALUES 
    (1,'admin','ADMIN_HASH','Admin','Admin',1,1,'2020-08-17 18:03:20','',NULL,'2020-04-26 13:08:34','2020-04-26 13:10:45','User',NULL,'all','ADMIN_SALT',0,'2020-04-26 13:10:27'),
    (2,'','','','Anonymous users',0,1,NULL,'',NULL,'2020-04-26 13:08:38','2020-04-26 13:08:38','GroupAnonymous',NULL,'',NULL,0,NULL),
    (3,'','','','Non member users',0,1,NULL,'',NULL,'2020-04-26 13:08:38','2020-04-26 13:08:38','GroupNonMember',NULL,'',NULL,0,NULL),
    (4,'','','','Anonymous',0,0,NULL,'',NULL,'2020-04-26 13:09:44','2020-04-26 13:09:44','AnonymousUser',NULL,'only_my_events',NULL,0,NULL),
    (5,'','','','Analysts',0,1,NULL,'',NULL,'2020-04-26 18:43:40','2020-04-26 18:43:40','Group',NULL,'',NULL,0,NULL),
    (6,'','','','Automation',0,1,NULL,'',NULL,'2020-04-26 18:43:47','2020-04-26 18:43:47','Group',NULL,'',NULL,0,NULL),
    (7,'','','','Admins',0,1,NULL,'',NULL,'2020-04-26 18:43:58','2020-04-26 18:43:58','Group',NULL,'',NULL,0,NULL)
 ;
 /*!40000 ALTER TABLE `users` ENABLE KEYS */;
 UNLOCK TABLES;
@@ -9,7 +9,7 @@
 {% set MANAGER = salt['grains.get']('master') %}
 {% set MAINIP = salt['grains.get']('ip_interfaces').get(salt['pillar.get']('sensor:mainint', salt['pillar.get']('manager:mainint', salt['pillar.get']('elasticsearch:mainint', salt['pillar.get']('host:mainint')))))[0] %}
 {%- set MYSQLPASS = salt['pillar.get']('secrets:mysql', None) -%}
-{%- set PLAYBOOKPASS = salt['pillar.get']('secrets:playbook', None) -%}
+{%- set PLAYBOOKPASS = salt['pillar.get']('secrets:playbook_db', None) -%}
 include:
  - mysql
@@ -58,6 +58,14 @@ query_updatepluginurls:
    - connection_user: root
    - connection_pass: {{ MYSQLPASS }}
 playbooklogdir:
  file.directory:
    - name: /opt/so/log/playbook
    - dir_mode: 775
    - user: 939
    - group: 939
    - makedirs: True
 {% if PLAYBOOKPASS == None %}
 playbook_password_none:
@@ -73,6 +81,8 @@ so-playbook:
    - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-playbook:{{ VERSION }}
    - hostname: playbook
    - name: so-playbook
    - binds:
      - /opt/so/log/playbook:/playbook/log:rw
    - environment:
      - REDMINE_DB_MYSQL={{ MANAGERIP }}
      - REDMINE_DB_DATABASE=playbook
@@ -83,13 +93,6 @@ so-playbook:
 {% endif %}
 playbooklogdir:
  file.directory:
    - name: /opt/so/log/playbook
    - user: 939
    - group: 939
    - makedirs: True
 so-playbooksynccron:
  cron.present:
    - name: /usr/sbin/so-playbook-sync > /opt/so/log/playbook/sync.log 2>&1
--- a/Show More
+++ b/Show More
		`@@ -0,0 +1,2 @@`
							`#!/bin/bash`
							`logrotate -f /opt/so/conf/log-rotate.conf >/dev/null 2>&1`
		`@@ -0,0 +1 @@`
							<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 87.86 105.22"><defs><style>.cls-1{fill:#fff;}.cls-2{fill:#1976d2;}</style></defs><g id="Layer_2" data-name="Layer 2"><g id="Layer_1-2" data-name="Layer 1"><g id="Onion"><path id="Flesh" class="cls-1" d="M43.37,71.34a1.27,1.27,0,0,0,.44-.51,4.74,4.74,0,0,0,.61-2.39c-.12-6.79-.22-12.88-4-14.46-4.05-1.72-9.38,3.14-10.71,4.35a19.84,19.84,0,0,0-6.17,12.34c-.1,1-.76,9.34,5.46,15.41s15.45,6.06,21.72,3.53A22.25,22.25,0,0,0,61.88,79.16c5.31-10,1.61-20.31.85-22.3C57.78,44,43.35,36.11,29.88,36.78c-2.17.11-15.82,1-24.16,12.42A30.55,30.55,0,0,0,0,67.36c.15,16.14,13.38,29.51,26.23,34.7,12.61,5.1,24,2.76,28.78,1.65s17.12-4,25.53-15.08a34.47,34.47,0,0,0,7.24-18.46,34.79,34.79,0,0,0-3.42-17.32c-1.11-2.3-6.16-12.09-17-17C57,31.21,48.52,34.37,45.65,29.12a8.46,8.46,0,0,1-.41-6.21,1,1,0,0,0-1.05-1.28l-1.6,0a1.07,1.07,0,0,0-1,.8c-.66,2.51-1.12,6,.51,9.17C46,39.08,56.87,35.31,67.56,42.78c8.29,5.79,14.14,16.69,13.21,27.29a28.06,28.06,0,0,1-6,14.65c-7,9-17,11.29-21.82,12.38-4,.9-13.19,2.87-23.54-.93-2.65-1-20.33-8.29-22.38-25C5.72,60.55,13,48.9,24.21,44.93c13-4.6,27.26,2.75,32.09,13.26.58,1.25,4.85,10.93-.59,18.72-4.05,5.79-13.07,9.94-19.77,6A13.48,13.48,0,0,1,30,68.25c1.42-5,6.37-8.72,8.13-7.84s2.94,6.14,3,9.85A1.39,1.39,0,0,0,43.37,71.34Z"/><path id="Stem" class="cls-2" d="M30,27.14l-4.17,1.27a1.16,1.16,0,0,1-1.49-.93l-.11-.72a26.93,26.93,0,0,0-4.53-11.09A1.13,1.13,0,0,1,20.06,14l1.06-.63a1.15,1.15,0,0,1,1.52.32c.41.58.82,1.17,1.23,1.78l1.48,2.2C28.42,7.27,37.14.12,46.21,0,58.09-.16,65.59,10.67,68,17.63a23.37,23.37,0,0,1,.94,3.64.91.91,0,0,1-1.14,1l-2.66-.73a1.47,1.47,0,0,1-1-1.08,19.71,19.71,0,0,0-1.9-4.8c-3-5.44-9.67-11.21-16.55-10.59-7.74.7-15.22,9.46-14.85,20.91A1.14,1.14,0,0,1,30,27.14Z"/></g></g></g></svg>