Merge pull request #4411 from Security-Onion-Solutions/hotfix-0528

2.3.52

HOTFIX

@@ -0,0 +1 @@
+

README.md

@@ -1,6 +1,6 @@
-## Security Onion 2.3.30
+## Security Onion 2.3.52
 
-Security Onion 2.3.30 is here!
+Security Onion 2.3.52 is here!
 
 ## Screenshots
 

VERIFY_ISO.md

@@ -1,16 +1,17 @@
-### 2.3.30 ISO image built on 2021/03/01
+### 2.3.52 ISO image built on 2021/04/27
+
 
 ### Download and Verify
 
-2.3.30 ISO image:  
-https://download.securityonion.net/file/securityonion/securityonion-2.3.30.iso
+2.3.52 ISO image:  
+https://download.securityonion.net/file/securityonion/securityonion-2.3.52.iso
 
-MD5: 65202BA0F7661A5E27087F097B8E571E  
-SHA1: 14E842E39EDBB55A104263281CF25BF88A2E9D67  
-SHA256: 210B37B9E3DFC827AFE2940E2C87B175ADA968EDD04298A5926F63D9269847B7 
+MD5: DF0CCCB0331780F472CC167AEAB55652  
+SHA1: 71FAE87E6C0AD99FCC27C50A5E5767D3F2332260  
+SHA256: 30E7C4206CC86E94D1657CBE420D2F41C28BC4CC63C51F27C448109EBAF09121 
 
 Signature for ISO image:  
-https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.30.iso.sig
+https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.52.iso.sig
 
 Signing key:  
 https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS  
@@ -24,22 +25,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/ma
 
 Download the signature file for the ISO:  
 ```
-wget https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.30.iso.sig
+wget https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.52.iso.sig
 ```
 
 Download the ISO image:  
 ```
-wget https://download.securityonion.net/file/securityonion/securityonion-2.3.30.iso
+wget https://download.securityonion.net/file/securityonion/securityonion-2.3.52.iso
 ```
 
 Verify the downloaded ISO image using the signature file:  
 ```
-gpg --verify securityonion-2.3.30.iso.sig securityonion-2.3.30.iso
+gpg --verify securityonion-2.3.52.iso.sig securityonion-2.3.52.iso
 ```
 
 The output should show "Good signature" and the Primary key fingerprint should match what's shown below:
 ```
-gpg: Signature made Mon 01 Mar 2021 02:15:28 PM EST using RSA key ID FE507013
+gpg: Signature made Sat 05 Jun 2021 06:56:04 PM EDT using RSA key ID FE507013
 gpg: Good signature from "Security Onion Solutions, LLC <info@securityonionsolutions.com>"
 gpg: WARNING: This key is not certified with a trusted signature!
 gpg:          There is no indication that the signature belongs to the owner.

VERSION

@@ -1 +1 @@
-2.3.30
+2.3.52

pillar/docker/config.sls

@@ -1,208 +0,0 @@
-{%- set FLEETMANAGER = salt['pillar.get']('global:fleet_manager', False) -%}
-{%- set FLEETNODE = salt['pillar.get']('global:fleet_node', False) -%}
-{% set WAZUH = salt['pillar.get']('manager:wazuh', '0') %}
-{% set THEHIVE = salt['pillar.get']('manager:thehive', '0') %}
-{% set PLAYBOOK = salt['pillar.get']('manager:playbook', '0') %}
-{% set FREQSERVER = salt['pillar.get']('manager:freq', '0') %}
-{% set DOMAINSTATS = salt['pillar.get']('manager:domainstats', '0') %}
-{% set ZEEKVER = salt['pillar.get']('global:mdengine', 'COMMUNITY') %}
-{% set GRAFANA = salt['pillar.get']('manager:grafana', '0') %}
-
-eval:
-  containers:
-    - so-nginx
-    - so-telegraf
-    {% if  GRAFANA == '1' %}
-    - so-influxdb
-    - so-grafana
-    {% endif %}
-    - so-dockerregistry
-    - so-soc
-    - so-kratos
-    - so-idstools
-    {% if FLEETMANAGER %}
-    - so-mysql
-    - so-fleet
-    - so-redis
-    {% endif %}
-    - so-elasticsearch
-    - so-logstash
-    - so-kibana
-    - so-steno
-    - so-suricata
-    - so-zeek
-    - so-curator
-    - so-elastalert
-    {% if WAZUH != '0' %}
-    - so-wazuh
-    {% endif %}
-    - so-soctopus
-    {% if THEHIVE != '0' %}
-    - so-thehive
-    - so-thehive-es
-    - so-cortex
-    {% endif %}
-    {% if PLAYBOOK != '0' %}
-    - so-playbook
-    {% endif %}
-    {% if FREQSERVER != '0' %}
-    - so-freqserver
-    {% endif %}
-    {% if DOMAINSTATS != '0' %}
-    - so-domainstats
-    {% endif %}
-heavy_node:
-  containers:
-    - so-nginx
-    - so-telegraf
-    - so-redis
-    - so-logstash
-    - so-elasticsearch
-    - so-curator
-    - so-steno
-    - so-suricata
-    - so-wazuh
-    - so-filebeat
-    {% if ZEEKVER != 'SURICATA' %}
-    - so-zeek
-    {% endif %}
-helix:
-  containers:
-    - so-nginx
-    - so-telegraf
-    - so-idstools
-    - so-steno
-    - so-zeek
-    - so-redis
-    - so-logstash
-    - so-filebeat
-hot_node:
-  containers:
-    - so-nginx
-    - so-telegraf
-    - so-logstash
-    - so-elasticsearch
-    - so-curator
-manager_search:
-  containers:
-    - so-nginx
-    - so-telegraf
-    - so-soc
-    - so-kratos
-    - so-acng
-    - so-idstools
-    - so-redis
-    - so-logstash
-    - so-elasticsearch
-    - so-curator
-    - so-kibana
-    - so-elastalert
-    - so-filebeat
-    - so-soctopus
-    {% if FLEETMANAGER %}
-    - so-mysql
-    - so-fleet
-    - so-redis
-    {% endif %}
-    {% if WAZUH != '0' %}
-    - so-wazuh
-    {% endif %}
-    - so-soctopus
-    {% if THEHIVE != '0' %}
-    - so-thehive
-    - so-thehive-es
-    - so-cortex
-    {% endif %}
-    {% if PLAYBOOK != '0' %}
-    - so-playbook
-    {% endif %}
-    {% if FREQSERVER != '0' %}
-    - so-freqserver
-    {% endif %}
-    {% if DOMAINSTATS != '0' %}
-    - so-domainstats
-    {% endif %}
-manager:
-  containers:
-    - so-dockerregistry
-    - so-nginx
-    - so-telegraf
-    {% if  GRAFANA == '1' %}
-    - so-influxdb
-    - so-grafana
-    {% endif %}
-    - so-soc
-    - so-kratos
-    - so-acng
-    - so-idstools
-    - so-redis
-    - so-elasticsearch
-    - so-logstash
-    - so-kibana
-    - so-elastalert
-    - so-filebeat
-    {% if FLEETMANAGER %}
-    - so-mysql
-    - so-fleet
-    - so-redis
-    {% endif %}
-    {% if WAZUH != '0' %}
-    - so-wazuh
-    {% endif %}
-    - so-soctopus
-    {% if THEHIVE != '0' %}
-    - so-thehive
-    - so-thehive-es
-    - so-cortex
-    {% endif %}
-    {% if PLAYBOOK != '0' %}
-    - so-playbook
-    {% endif %}
-    {% if FREQSERVER != '0' %}
-    - so-freqserver
-    {% endif %}
-    {% if DOMAINSTATS != '0' %}
-    - so-domainstats
-    {% endif %}
-parser_node:
-  containers:
-    - so-nginx
-    - so-telegraf
-    - so-logstash
-search_node:
-  containers:
-    - so-nginx
-    - so-telegraf
-    - so-logstash
-    - so-elasticsearch
-    - so-curator
-    - so-filebeat
-    {% if WAZUH != '0' %}
-    - so-wazuh
-    {% endif %}
-sensor:
-  containers:
-    - so-nginx
-    - so-telegraf
-    - so-steno
-    - so-suricata
-    {% if ZEEKVER != 'SURICATA' %}
-    - so-zeek
-    {% endif %}
-    - so-wazuh
-    - so-filebeat
-warm_node:
-  containers:
-    - so-nginx
-    - so-telegraf
-    - so-elasticsearch
-fleet:
-  containers:
-    {% if FLEETNODE %}
-    - so-mysql
-    - so-fleet
-    - so-redis
-    - so-filebeat
-    - so-nginx
-    - so-telegraf
-    {% endif %}

pillar/logstash/search.sls

@@ -7,7 +7,7 @@ logstash:
         - so/9000_output_zeek.conf.jinja
         - so/9002_output_import.conf.jinja
         - so/9034_output_syslog.conf.jinja
-        - so/9100_output_osquery.conf.jinja
+        - so/9100_output_osquery.conf.jinja  
         - so/9400_output_suricata.conf.jinja
         - so/9500_output_beats.conf.jinja
         - so/9600_output_ossec.conf.jinja

salt/airgap/init.sls

@@ -1,71 +0,0 @@
-{% from 'allowed_states.map.jinja' import allowed_states %}
-{% if sls in allowed_states %}
-
-{% set MANAGER = salt['grains.get']('master') %}
-airgapyum:
-  file.managed:
-    - name: /etc/yum/yum.conf
-    - source: salt://airgap/files/yum.conf
-
-airgap_repo:
-  pkgrepo.managed:
-    - humanname: Airgap Repo
-    - baseurl: https://{{ MANAGER }}/repo
-    - gpgcheck: 0
-    - sslverify: 0
-
-agbase:
-  file.absent:
-    - name: /etc/yum.repos.d/CentOS-Base.repo
-
-agcr:
-  file.absent:
-    - name: /etc/yum.repos.d/CentOS-CR.repo
-
-agdebug:
-  file.absent:
-    - name: /etc/yum.repos.d/CentOS-Debuginfo.repo
-
-agfasttrack:
-  file.absent:
-    - name: /etc/yum.repos.d/CentOS-fasttrack.repo
-
-agmedia:
-  file.absent:
-    - name: /etc/yum.repos.d/CentOS-Media.repo
-
-agsources:
-  file.absent:
-    - name: /etc/yum.repos.d/CentOS-Sources.repo
-
-agvault:
-  file.absent:
-    - name: /etc/yum.repos.d/CentOS-Vault.repo
-
-agkernel:
-  file.absent:
-    - name: /etc/yum.repos.d/CentOS-x86_64-kernel.repo
-
-agepel:
-  file.absent:
-    - name: /etc/yum.repos.d/epel.repo
-
-agtesting:
-  file.absent:
-    - name: /etc/yum.repos.d/epel-testing.repo
-
-agssrepo:
-  file.absent:
-    - name: /etc/yum.repos.d/saltstack.repo
-
-agwazrepo:
-  file.absent:
-    - name: /etc/yum.repos.d/wazuh.repo
-
-{% else %}
-
-{{sls}}_state_not_allowed:
-  test.fail_without_changes:
-    - name: {{sls}}_state_not_allowed
-
-{% endif %}

salt/ca/init.sls

@@ -43,8 +43,9 @@ pki_private_key:
     - require:
       - file: /etc/pki
     - timeout: 30
-    - retry: 5
-    - interval: 30
+    - retry:
+        attempts: 5
+        interval: 30
 
 x509_pem_entries:
   module.run:

salt/common/files/99-reserved-ports.conf

@@ -1 +1 @@
-net.ipv4.ip_local_reserved_ports="55000,57314"
+net.ipv4.ip_local_reserved_ports=55000,57314,47760-47860

salt/common/files/soversion

@@ -0,0 +1,2 @@
+{%- set VERSION = salt['pillar.get']('global:soversion') -%}
+{{ VERSION }}

salt/common/files/vimrc

@@ -0,0 +1,6 @@
+" Activates filetype detection
+filetype plugin indent on
+
+" Sets .sls files to use YAML syntax highlighting
+autocmd BufNewFile,BufRead *.sls set syntax=yaml
+set number

salt/common/init.sls

@@ -49,6 +49,11 @@ sosaltstackperms:
     - gid: 939
     - dir_mode: 770
 
+so_log_perms:
+  file.directory:
+    - name: /opt/so/log
+    - dir_mode: 755
+
 # Create a state directory
 statedir:
   file.directory:
@@ -64,20 +69,12 @@ salttmp:
     - group: 939
     - makedirs: True
 
-# Install epel
-{% if grains['os'] == 'CentOS' %}
-repair_yumdb:
-  cmd.run:
-    - name: 'mv -f /var/lib/rpm/__db* /tmp && yum clean all'
-    - onlyif:
-      - 'yum check-update 2>&1 | grep "Error: rpmdb open failed"'
-
-epel:
-  pkg.installed:
-    - skip_suggestions: True
-    - pkgs:
-      - epel-release
-{% endif %}
+# VIM config
+vimconfig:
+  file.managed:
+    - name: /root/.vimrc
+    - source: salt://common/files/vimrc
+    - replace: False
 
 # Install common packages
 {% if grains['os'] != 'CentOS' %}     
@@ -90,7 +87,6 @@ commonpkgs:
       - ntpdate
       - jq
       - python3-docker
-      - docker-ce
       - curl
       - ca-certificates
       - software-properties-common
@@ -104,12 +100,17 @@ commonpkgs:
       - python3-dateutil
       - python3-m2crypto
       - python3-mysqldb
+      - python3-packaging
       - git
+      - vim
+
 heldpackages:
   pkg.installed:
     - pkgs:
-      - containerd.io: 1.2.13-2
-      - docker-ce: 5:19.03.14~3-0~ubuntu-bionic
+      - containerd.io: 1.4.4-1
+      - docker-ce: 5:20.10.5~3-0~ubuntu-bionic
+      - docker-ce-cli: 5:20.10.5~3-0~ubuntu-bionic
+      - docker-ce-rootless-extras: 5:20.10.5~3-0~ubuntu-bionic
     - hold: True
     - update_holds: True
 
@@ -135,17 +136,21 @@ commonpkgs:
       - python36-dateutil
       - python36-m2crypto
       - python36-mysql
+      - python36-packaging
       - yum-utils
       - device-mapper-persistent-data
       - lvm2
       - openssl
       - git
+      - vim-enhanced
 
 heldpackages:
   pkg.installed:
     - pkgs:
-      - containerd.io: 1.2.13-3.2.el7
-      - docker-ce: 3:19.03.14-3.el7
+      - containerd.io: 1.4.4-3.1.el7
+      - docker-ce: 3:20.10.5-3.el7
+      - docker-ce-cli: 1:20.10.5-3.el7
+      - docker-ce-rootless-extras: 20.10.5-3.el7
     - hold: True
     - update_holds: True
 {% endif %}
@@ -230,6 +235,30 @@ commonlogrotateconf:
     - month: '*'
     - dayweek: '*'
 
+# Create the status directory
+sostatusdir:
+  file.directory:
+    - name: /opt/so/log/sostatus
+    - user: 0
+    - group: 0
+    - makedirs: True
+
+sostatus_log:
+  file.managed:
+    - name: /opt/so/log/sostatus/status.log
+    - mode: 644
+    
+# Install sostatus check cron
+'/usr/sbin/so-status -q; echo $? > /opt/so/log/sostatus/status.log 2>&1':
+  cron.present:
+    - user: root
+    - minute: '*/1'
+    - hour: '*'
+    - daymonth: '*'
+    - month: '*'
+    - dayweek: '*'
+
+
 {% if role in ['eval', 'manager', 'managersearch', 'standalone'] %}
 # Lock permissions on the backup directory
 backupdir:
@@ -249,6 +278,14 @@ backupdir:
     - daymonth: '*'
     - month: '*'
     - dayweek: '*'
+{% else %}
+soversionfile:
+  file.managed:
+    - name: /etc/soversion
+    - source: salt://common/files/soversion
+    - mode: 644
+    - template: jinja
+    
 {% endif %}
 
 # Manager daemon.json
@@ -266,9 +303,10 @@ docker:
       - file: docker_daemon
 
 # Reserve OS ports for Docker proxy in case boot settings are not already applied/present
+# 55000 = Wazuh, 57314 = Strelka, 47760-47860 = Zeek
 dockerapplyports:
     cmd.run:
-      - name: if [ ! -s /etc/sysctl.d/99-reserved-ports.conf ]; then sysctl -w net.ipv4.ip_local_reserved_ports="55000,57314"; fi
+      - name: if [ ! -s /etc/sysctl.d/99-reserved-ports.conf ]; then sysctl -w net.ipv4.ip_local_reserved_ports="55000,57314,47760-47860"; fi
 
 # Reserve OS ports for Docker proxy
 dockerreserveports:

salt/common/tools/sbin/so-airgap-hotfixapply

@@ -0,0 +1,64 @@
+#!/bin/bash
+
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+. /usr/sbin/so-common
+
+UPDATE_DIR=/tmp/sohotfixapply
+
+if [ -z "$1" ]; then
+  echo "No tarball given. Please provide the filename so I can run the hotfix"
+  echo "so-airgap-hotfixapply /path/to/sohotfix.tar" 
+  exit 1
+else
+  if [ ! -f "$1" ]; then
+    echo "Unable to find $1. Make sure your path is correct and retry."
+    exit 1
+  else
+    echo "Determining if we need to apply this hotfix"
+    rm -rf $UPDATE_DIR
+    mkdir -p $UPDATE_DIR
+    tar xvf $1 -C $UPDATE_DIR
+
+    # Compare some versions
+    NEWVERSION=$(cat $UPDATE_DIR/VERSION)
+    HOTFIXVERSION=$(cat $UPDATE_DIR/HOTFIX)
+    CURRENTHOTFIX=$(cat /etc/sohotfix)
+    INSTALLEDVERSION=$(cat /etc/soversion)
+
+    if [ "$INSTALLEDVERSION" == "$NEWVERSION" ]; then
+      echo "Checking to see if there are hotfixes needed"
+      if [ "$HOTFIXVERSION" == "$CURRENTHOTFIX" ]; then
+        echo "You are already running the latest version of Security Onion."
+        rm -rf $UPDATE_DIR
+        exit 1
+      else
+        echo "We need to apply a hotfix"
+        copy_new_files
+        echo $HOTFIXVERSION > /etc/sohotfix
+        salt-call state.highstate -l info queue=True
+        echo "The Hotfix $HOTFIXVERSION has been applied"
+        # Clean up 
+        rm -rf $UPDATE_DIR
+        exit 0
+      fi
+    else
+      echo "This hotfix is not compatible with your current version. Download the latest ISO and run soup"
+      rm -rf $UPDATE_DIR
+    fi
+
+  fi
+fi

salt/common/tools/sbin/so-airgap-hotfixdownload

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# Get the latest code
+rm -rf /tmp/sohotfix
+mkdir -p /tmp/sohotfix
+cd /tmp/sohotfix
+git clone https://github.com/Security-Onion-Solutions/securityonion
+if [ ! -d "/tmp/sohotfix/securityonion" ]; then
+  echo "I was unable to get the latest code. Check your internet and try again."
+  exit 1
+else
+  echo "Looks like we have the code lets create the tarball."
+  cd /tmp/sohotfix/securityonion
+  tar cvf /tmp/sohotfix/sohotfix.tar HOTFIX VERSION salt pillar
+  echo ""
+  echo "Copy /tmp/sohotfix/sohotfix.tar to portable media and then copy it to your airgap manager."
+  exit 0
+fi

salt/common/tools/sbin/so-common

@@ -15,6 +15,8 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+DEFAULT_SALT_DIR=/opt/so/saltstack/default
+
 # Check for prerequisites
 if [ "$(id -u)" -ne 0 ]; then
 	echo "This script must be run using sudo!"
@@ -86,6 +88,19 @@ add_interface_bond0() {
 	fi
 }
 
+check_airgap() {
+  # See if this is an airgap install
+  AIRGAP=$(cat /opt/so/saltstack/local/pillar/global.sls | grep airgap: | awk '{print $2}')
+  if [[ "$AIRGAP" == "True" ]]; then
+      is_airgap=0
+      UPDATE_DIR=/tmp/soagupdate/SecurityOnion
+      AGDOCKER=/tmp/soagupdate/docker
+      AGREPO=/tmp/soagupdate/Packages
+  else 
+      is_airgap=1
+  fi
+}
+
 check_container() {
 	docker ps | grep "$1:" > /dev/null 2>&1
 	return $?
@@ -97,6 +112,60 @@ check_password() {
 	return $?
 }
 
+check_elastic_license() {
+
+	[ -n "$TESTING" ] && return
+
+	# See if the user has already accepted the license
+	if [ ! -f /opt/so/state/yeselastic.txt ]; then
+	  elastic_license
+	else
+	  echo "Elastic License has already been accepted"
+	fi  
+}
+
+copy_new_files() {
+  # Copy new files over to the salt dir
+  cd $UPDATE_DIR
+  rsync -a salt $DEFAULT_SALT_DIR/
+  rsync -a pillar $DEFAULT_SALT_DIR/
+  chown -R socore:socore $DEFAULT_SALT_DIR/
+  chmod 755 $DEFAULT_SALT_DIR/pillar/firewall/addfirewall.sh
+  cd /tmp
+}
+
+disable_fastestmirror() {
+	sed -i 's/enabled=1/enabled=0/' /etc/yum/pluginconf.d/fastestmirror.conf
+}
+
+elastic_license() {
+
+read -r -d '' message <<- EOM
+\n
+Starting in Elastic Stack version 7.11, the Elastic Stack binaries are only available under the Elastic License:
+https://securityonion.net/elastic-license
+
+Please review the Elastic License:
+https://www.elastic.co/licensing/elastic-license
+
+Do you agree to the terms of the Elastic License?
+
+If so, type AGREE to accept the Elastic License and continue.  Otherwise, press Enter to exit this program without making any changes.
+EOM
+
+AGREED=$(whiptail --title "Security Onion Setup" --inputbox \
+"$message" 20 75 3>&1 1>&2 2>&3)
+
+if [ "${AGREED^^}" = 'AGREE' ]; then
+  mkdir -p /opt/so/state
+  touch /opt/so/state/yeselastic.txt 
+else
+  echo "Starting in 2.3.40 you must accept the Elastic license if you want to run Security Onion."
+  exit 1
+fi
+
+}
+
 fail() {
 	msg=$1
 	echo "ERROR: $msg"
@@ -109,6 +178,23 @@ get_random_value() {
 	head -c 5000 /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w $length | head -n 1
 }
 
+gpg_rpm_import() {
+	if [[ "$OS" == "centos" ]]; then
+	    if [[ "$WHATWOULDYOUSAYYAHDOHERE" == "setup" ]]; then
+	        local RPMKEYSLOC="../salt/repo/client/files/centos/keys"
+	    else
+	        local RPMKEYSLOC="$UPDATE_DIR/salt/repo/client/files/centos/keys"
+	    fi
+	
+	    RPMKEYS=('RPM-GPG-KEY-EPEL-7' 'GPG-KEY-WAZUH' 'docker.pub' 'SALTSTACK-GPG-KEY.pub' 'securityonion.pub')
+
+	    for RPMKEY in "${RPMKEYS[@]}"; do
+    	        rpm --import $RPMKEYSLOC/$RPMKEY
+	        echo "Imported $RPMKEY"
+	    done
+	fi
+}
+
 header() {
 	printf '%s\n' "" "$banner" " $*" "$banner" 
 }
@@ -250,6 +336,12 @@ set_minionid() {
 	MINIONID=$(lookup_grain id)
 }
 
+set_palette() {
+	if [ "$OS" == ubuntu ]; then
+	    update-alternatives --set newt-palette /etc/newt/palette.original
+    fi
+}
+
 set_version() {
 	CURRENTVERSION=0.0.0
 	if [ -f /etc/soversion ]; then
@@ -340,6 +432,40 @@ valid_int() {
 
 # {% raw %}
 
+valid_proxy() {
+	local proxy=$1
+	local url_prefixes=( 'http://' 'https://' )
+
+	local has_prefix=false
+	for prefix in "${url_prefixes[@]}"; do
+		echo "$proxy" | grep -q "$prefix" && has_prefix=true && proxy=${proxy#"$prefix"} && break
+	done
+	
+	local url_arr
+	mapfile -t url_arr <<< "$(echo "$proxy" | tr ":" "\n")"
+
+	local valid_url=true
+	if ! valid_ip4 "${url_arr[0]}" && ! valid_fqdn "${url_arr[0]}" && ! valid_hostname "${url_arr[0]}"; then
+		valid_url=false
+	fi
+
+	[[ $has_prefix == true ]] && [[ $valid_url == true ]] && return 0 || return 1
+}
+
+valid_ntp_list() {
+	local string=$1
+	local ntp_arr
+	IFS="," read -r -a ntp_arr <<< "$string"
+
+	for ntp in "${ntp_arr[@]}"; do
+		if ! valid_ip4 "$ntp" && ! valid_hostname "$ntp" && ! valid_fqdn "$ntp"; then
+			return 1
+		fi
+	done
+
+	return 0
+}
+
 valid_string() {
 	local str=$1
 	local min_length=${2:-1}
@@ -361,6 +487,7 @@ wait_for_web_response() {
 	expected=$2
 	maxAttempts=${3:-300}
 	logfile=/root/wait_for_web_response.log
+	truncate -s 0 "$logfile"
 	attempt=0
 	while [[ $attempt -lt $maxAttempts ]]; do
 		attempt=$((attempt+1))

salt/common/tools/sbin/so-cortex-user-add

@@ -30,7 +30,7 @@ fi
 
 USER=$1
 
-CORTEX_KEY=$(lookup_pillar cortexkey)
+CORTEX_KEY=$(lookup_pillar cortexorguserkey)
 CORTEX_API_URL="$(lookup_pillar url_base)/cortex/api"
 CORTEX_ORG_NAME=$(lookup_pillar cortexorgname)
 CORTEX_USER=$USER

salt/common/tools/sbin/so-cortex-user-enable

@@ -30,7 +30,7 @@ fi
 
 USER=$1
 
-CORTEX_KEY=$(lookup_pillar cortexkey)
+CORTEX_KEY=$(lookup_pillar cortexorguserkey)
 CORTEX_API_URL="$(lookup_pillar url_base)/cortex/api"
 CORTEX_USER=$USER
 

salt/common/tools/sbin/so-docker-prune

@@ -0,0 +1,89 @@
+#!/usr/bin/env python3
+
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import sys, argparse, re, docker
+from packaging.version import Version, InvalidVersion
+from itertools import groupby, chain
+
+
+def get_image_name(string) -> str:
+  return ':'.join(string.split(':')[:-1])
+
+
+def get_so_image_basename(string) -> str:
+  return get_image_name(string).split('/so-')[-1]
+
+
+def get_image_version(string) -> str:
+  ver = string.split(':')[-1]
+  if ver == 'latest':
+    # Version doesn't like "latest", so use a high semver
+    return '999999.9.9'
+  else:
+    try:
+      Version(ver)
+    except InvalidVersion:
+      # Strip the last substring following a hyphen for automated branches
+      ver = '-'.join(ver.split('-')[:-1])  
+    return ver
+
+
+def main(quiet):
+  client = docker.from_env()
+
+  image_list = client.images.list(filters={ 'dangling': False })
+
+  # Map list of image objects to flattened list of tags (format: "name:version")
+  tag_list = list(chain.from_iterable(list(map(lambda x: x.attrs.get('RepoTags'), image_list))))
+
+  # Filter to only SO images (base name begins with "so-")
+  tag_list = list(filter(lambda x: re.match(r'^.*\/so-[^\/]*$', get_image_name(x)), tag_list))
+
+  # Group tags into lists by base name (sort by same projection first)
+  tag_list.sort(key=lambda x: get_so_image_basename(x))
+  grouped_tag_lists = [ list(it) for _, it in groupby(tag_list, lambda x: get_so_image_basename(x)) ]
+
+  no_prunable = True
+  for t_list in grouped_tag_lists:
+    try:
+      # Group tags by version, in case multiple images exist with the same version string
+      t_list.sort(key=lambda x: Version(get_image_version(x)), reverse=True)
+      grouped_t_list = [ list(it) for _,it in groupby(t_list, lambda x: get_image_version(x)) ]
+
+      # Keep the 2 most current version groups
+      if len(grouped_t_list) <= 2:
+        continue
+      else:
+        no_prunable = False
+        for group in grouped_t_list[2:]:
+          for tag in group:
+            if not quiet: print(f'Removing image {tag}')
+            client.images.remove(tag)
+    except InvalidVersion as e:
+      print(f'so-{get_so_image_basename(t_list[0])}: {e.args[0]}', file=sys.stderr)
+      exit(1)
+
+  if no_prunable and not quiet:
+    print('No Security Onion images to prune')
+
+
+if __name__ == "__main__":
+  main_parser = argparse.ArgumentParser(add_help=False)
+  main_parser.add_argument('-q', '--quiet', action='store_const', const=True, required=False)
+  args = main_parser.parse_args(sys.argv[1:])
+ 
+  main(args.quiet)

salt/common/tools/sbin/so-elastic-clear

@@ -50,11 +50,7 @@ done
 if [ $SKIP -ne 1 ]; then
         # List indices
         echo 
-        {% if grains['role'] in ['so-node','so-heavynode'] %}
         curl -k -L https://{{ NODEIP }}:9200/_cat/indices?v
-        {% else %}
-        curl -L {{ NODEIP }}:9200/_cat/indices?v
-        {% endif %}
         echo
         # Inform user we are about to delete all data
         echo
@@ -93,18 +89,10 @@ fi
 # Delete data
 echo "Deleting data..."
 
-{% if grains['role'] in ['so-node','so-heavynode'] %}
 INDXS=$(curl -s -XGET -k -L https://{{ NODEIP }}:9200/_cat/indices?v | egrep 'logstash|elastalert|so-' | awk '{ print $3 }')
-{% else %}
-INDXS=$(curl -s -XGET -L {{ NODEIP }}:9200/_cat/indices?v | egrep 'logstash|elastalert|so-' | awk '{ print $3 }')
-{% endif %}
 for INDX in ${INDXS}
 do
-    {% if grains['role'] in ['so-node','so-heavynode'] %}
     curl -XDELETE -k -L https://"{{ NODEIP }}:9200/${INDX}" > /dev/null 2>&1
-    {% else %}
-    curl -XDELETE -L "{{ NODEIP }}:9200/${INDX}" > /dev/null 2>&1
-    {% endif %}
 done
 
 #Start Logstash/Filebeat

salt/common/tools/sbin/so-elasticsearch-indices-list

@@ -0,0 +1,21 @@
+#!/bin/bash
+#
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>
+{%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%}
+
+. /usr/sbin/so-common
+
+curl -s -k -L https://{{ NODEIP }}:9200/_cat/indices?pretty

salt/common/tools/sbin/so-elasticsearch-indices-rw

@@ -21,6 +21,5 @@ THEHIVEESPORT=9400
 
 echo "Removing read only attributes for indices..."
 echo
-for p in $ESPORT $THEHIVEESPORT; do
-   curl -XPUT -H "Content-Type: application/json" -L http://$IP:$p/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}' 2>&1 | if grep -q ack; then echo "Index settings updated..."; else echo "There was any issue updating the read-only attribute.  Please ensure Elasticsearch is running.";fi;
-done
+curl -s -k -XPUT -H "Content-Type: application/json" -L https://$IP:9200/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}' 2>&1 | if grep -q ack; then echo "Index settings updated..."; else echo "There was any issue updating the read-only attribute.  Please ensure Elasticsearch is running.";fi;
+curl -XPUT -H "Content-Type: application/json" -L http://$IP:9400/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}' 2>&1 | if grep -q ack; then echo "Index settings updated..."; else echo "There was any issue updating the read-only attribute.  Please ensure Elasticsearch is running.";fi;

salt/common/tools/sbin/so-elasticsearch-pipeline-stats

@@ -19,15 +19,7 @@
 . /usr/sbin/so-common
 
 if [ "$1" == "" ]; then
-        {% if grains['role'] in ['so-node','so-heavynode'] %}
         curl -s -k -L https://{{ NODEIP }}:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines"
-        {% else %}
-        curl -s -L {{ NODEIP }}:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines"
-        {% endif %}
 else
-        {% if grains['role'] in ['so-node','so-heavynode'] %}
         curl -s -k -L https://{{ NODEIP }}:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines.\"$1\""
-        {% else %}
-        curl -s -L {{ NODEIP }}:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines.\"$1\""
-        {% endif %}
 fi

salt/common/tools/sbin/so-elasticsearch-pipeline-view

@@ -0,0 +1,25 @@
+#!/bin/bash
+#
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>
+{%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%}
+
+. /usr/sbin/so-common
+
+if [ "$1" == "" ]; then
+        curl -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/* | jq .
+else
+        curl -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/$1 | jq .
+fi

salt/common/tools/sbin/so-elasticsearch-pipelines-list

@@ -17,15 +17,7 @@
 {%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%}
 . /usr/sbin/so-common
 if [ "$1" == "" ]; then
-    {% if grains['role'] in ['so-node','so-heavynode'] %}
     curl -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/* | jq 'keys'
-    {% else %}
-    curl -s -L {{ NODEIP }}:9200/_ingest/pipeline/* | jq 'keys'
-    {% endif %}
 else
-    {% if grains['role'] in ['so-node','so-heavynode'] %}
     curl -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/$1 | jq
-    {% else %}
-    curl -s -L {{ NODEIP }}:9200/_ingest/pipeline/$1 | jq
-    {% endif %}
 fi

salt/common/tools/sbin/so-elasticsearch-shards-list

@@ -0,0 +1,21 @@
+#!/bin/bash
+#
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>
+{%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%}
+
+. /usr/sbin/so-common
+
+curl -s -k -L https://{{ NODEIP }}:9200/_cat/shards?pretty

salt/common/tools/sbin/so-elasticsearch-template-remove

@@ -0,0 +1,21 @@
+#!/bin/bash
+#
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>
+{%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%}
+
+. /usr/sbin/so-common
+
+curl -s -k -L -XDELETE https://{{ NODEIP }}:9200/_template/$1

salt/common/tools/sbin/so-elasticsearch-template-view

@@ -0,0 +1,25 @@
+#!/bin/bash
+#
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>
+{%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%}
+
+. /usr/sbin/so-common
+
+if [ "$1" == "" ]; then
+        curl -s -k -L https://{{ NODEIP }}:9200/_template/* | jq .
+else
+        curl -s -k -L https://{{ NODEIP }}:9200/_template/$1 | jq .
+fi

salt/common/tools/sbin/so-elasticsearch-templates-list

@@ -17,15 +17,7 @@
 {%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%}
 . /usr/sbin/so-common
 if [ "$1" == "" ]; then
-    {% if grains['role'] in ['so-node','so-heavynode'] %}
     curl -s -k -L https://{{ NODEIP }}:9200/_template/* | jq 'keys'
-    {% else %}
-    curl -s -L {{ NODEIP }}:9200/_template/* | jq 'keys'
-    {% endif %}
 else
-    {% if grains['role'] in ['so-node','so-heavynode'] %}
     curl -s -k -L https://{{ NODEIP }}:9200/_template/$1 | jq
-    {% else %}
-    curl -s -L {{ NODEIP }}:9200/_template/$1 | jq
-    {% endif %}
 fi

salt/common/tools/sbin/so-elasticsearch-templates-load

@@ -30,11 +30,7 @@ echo -n "Waiting for ElasticSearch..."
 COUNT=0
 ELASTICSEARCH_CONNECTED="no"
 while [[ "$COUNT" -le 240 ]]; do
-    {% if grains['role'] in ['so-node','so-heavynode'] %}
     curl -k --output /dev/null --silent --head --fail -L https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"
-    {% else %}
-	curl --output /dev/null --silent --head --fail -L http://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"
-	{% endif %}
 	if [ $? -eq 0 ]; then
 		ELASTICSEARCH_CONNECTED="yes"
 		echo "connected!"
@@ -55,11 +51,7 @@ cd ${ELASTICSEARCH_TEMPLATES}
 
 
 echo "Loading templates..."
-{% if grains['role'] in ['so-node','so-heavynode'] %}
 for i in *; do TEMPLATE=$(echo $i | cut -d '-' -f2); echo "so-$TEMPLATE"; curl -k ${ELASTICSEARCH_AUTH} -s -XPUT -L https://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_template/so-$TEMPLATE -H 'Content-Type: application/json' -d@$i 2>/dev/null; echo; done
-{% else %}
-for i in *; do TEMPLATE=$(echo $i | cut -d '-' -f2); echo "so-$TEMPLATE"; curl ${ELASTICSEARCH_AUTH} -s -XPUT -L http://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_template/so-$TEMPLATE -H 'Content-Type: application/json' -d@$i 2>/dev/null; echo; done
-{% endif %}
 echo
 
 cd - >/dev/null

salt/common/tools/sbin/so-features-enable

@@ -1,53 +0,0 @@
-#!/bin/bash
-# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
-
-#    This program is free software: you can redistribute it and/or modify
-#    it under the terms of the GNU General Public License as published by
-#    the Free Software Foundation, either version 3 of the License, or
-#    (at your option) any later version.
-#
-#    This program is distributed in the hope that it will be useful,
-#    but WITHOUT ANY WARRANTY; without even the implied warranty of
-#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#    GNU General Public License for more details.
-#
-#    You should have received a copy of the GNU General Public License
-#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-. /usr/sbin/so-common
-. /usr/sbin/so-image-common
-local_salt_dir=/opt/so/saltstack/local
-
-cat << EOF
-This program will switch from the open source version of the Elastic Stack to the Features version licensed under the Elastic license.
-If you proceed, then we will download new Docker images and restart services.
-
-Please review the Elastic license:
-https://raw.githubusercontent.com/elastic/elasticsearch/master/licenses/ELASTIC-LICENSE.txt
-
-Please also note that, if you have a distributed deployment and continue with this change, Elastic traffic between nodes will change from encrypted to cleartext!
-(We expect to support Elastic Features Security at some point in the future.)
-
-Do you agree to the terms of the Elastic license and understand the note about encryption?
-
-If so, type AGREE to accept the Elastic license and continue.  Otherwise, just press Enter to exit this program without making any changes.
-EOF
-
-read INPUT
-if [ "$INPUT" != "AGREE" ]; then
-        exit
-fi
-
-echo "Please wait while switching to Elastic Features."
-
-require_manager
-
-TRUSTED_CONTAINERS=( \
-  "so-elasticsearch" \
-  "so-filebeat" \
-  "so-kibana" \
-  "so-logstash" )
-update_docker_containers "features" "-features"
-
-# Modify global.sls to enable Features
-sed -i 's/features: False/features: True/' $local_salt_dir/pillar/global.sls

salt/common/tools/sbin/so-image-common

@@ -47,20 +47,17 @@ container_list() {
     TRUSTED_CONTAINERS=(
       "so-acng"
       "so-curator"
-      "so-domainstats"
       "so-elastalert"
       "so-elasticsearch"
       "so-filebeat"
       "so-fleet"
       "so-fleet-launcher"
-      "so-freqserver"
       "so-grafana"
       "so-idstools"
       "so-influxdb"
       "so-kibana"
       "so-kratos"
       "so-logstash"
-      "so-minio"
       "so-mysql"
       "so-nginx"
       "so-pcaptools"

salt/common/tools/sbin/so-index-list

@@ -15,8 +15,4 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-{% if grains['role'] in ['so-node','so-heavynode'] %}
-curl -X GET -k -L https://localhost:9200/_cat/indices?v
-{% else %}
-curl -X GET -L localhost:9200/_cat/indices?v
-{% endif %}
+curl -X GET -k -L "https://localhost:9200/_cat/indices?v&s=index"

salt/common/tools/sbin/so-kibana-space-defaults

@@ -0,0 +1,13 @@
+. /usr/sbin/so-common
+
+wait_for_web_response "http://localhost:5601/app/kibana" "Elastic"
+## This hackery will be removed if using Elastic Auth ##
+
+# Let's snag a cookie from Kibana
+THECOOKIE=$(curl -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}')
+
+# Disable certain Features from showing up in the Kibana UI
+echo
+echo "Setting up default Space:"
+curl -b "sid=$THECOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["ml","enterpriseSearch","siem","logs","infrastructure","apm","uptime","monitoring","stackAlerts","actions","fleet"]} ' >> /opt/so/log/kibana/misc.log
+echo

salt/common/tools/sbin/so-logstash-events

@@ -0,0 +1,25 @@
+#!/bin/bash
+#
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>
+{%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%}
+
+. /usr/sbin/so-common
+
+if [ "$1" == "" ]; then
+        for i in $(curl -s -L http://{{ NODEIP }}:9600/_node/stats | jq .pipelines | jq '. | to_entries | .[].key' | sed 's/\"//g'); do echo ${i^}:; curl -s localhost:9600/_node/stats | jq .pipelines.$i.events; done
+else
+        curl -s -L http://{{ NODEIP }}:9600/_node/stats | jq .pipelines.$1.events
+fi

salt/common/tools/sbin/so-logstash-pipeline-stats

@@ -0,0 +1,25 @@
+#!/bin/bash
+#
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>
+{%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%}
+
+. /usr/sbin/so-common
+
+if [ "$1" == "" ]; then
+        curl -s -L http://{{ NODEIP }}:9600/_node/stats | jq .pipelines 
+else
+        curl -s -L http://{{ NODEIP }}:9600/_node/stats | jq .pipelines.$1
+fi

salt/common/tools/sbin/so-playbook-sync

@@ -17,4 +17,8 @@
 
 . /usr/sbin/so-common
 
+# Check to see if we are already running
+IS_RUNNING=$(ps aux | pgrep -f "so-playbook-sync" | wc -l)
+[ "$IS_RUNNING" -gt 3 ] && echo "$(date) - Multiple Playbook Sync processes already running...exiting." && exit 0
+
 docker exec so-soctopus python3 playbook_play-sync.py

salt/common/tools/sbin/so-raid-status

@@ -66,11 +66,13 @@ mkdir -p /opt/so/log/raid
   {%- if grains['sosmodel'] in ['SOSMN', 'SOSSNNV'] %}
 #check_boss_raid
 check_software_raid
-echo "osraid=$BOSSRAID nsmraid=$SWRAID" > /opt/so/log/raid/status.log
+#echo "osraid=$BOSSRAID nsmraid=$SWRAID" > /opt/so/log/raid/status.log
+echo "osraid=1 nsmraid=$SWRAID" > /opt/so/log/raid/status.log
   {%- elif grains['sosmodel'] in ['SOS1000F', 'SOS1000', 'SOSSN7200', 'SOS10K', 'SOS4000'] %}
 #check_boss_raid
 check_lsi_raid
-echo "osraid=$BOSSRAID nsmraid=$LSIRAID" > /opt/so/log/raid/status.log
+#echo "osraid=$BOSSRAID nsmraid=$LSIRAID" > /opt/so/log/raid/status.log
+echo "osraid=1 nsmraid=$LSIRAID" > /opt/so/log/raid/status.log
   {%- else %}
 exit 0
   {%- endif %}

salt/common/tools/sbin/so-rule

@@ -37,11 +37,9 @@ def print_err(string: str):
 
 
 def check_apply(args: dict, prompt: bool = True):
-  cmd_arr = ['salt-call', 'state.apply', 'idstools', 'queue=True']
-
   if args.apply:
-    print('Configuration updated. Applying idstools state...')
-    return subprocess.run(cmd_arr)
+    print('Configuration updated. Applying changes:')
+    return apply()
   else:
     if prompt:
       message = 'Configuration updated. Would you like to apply your changes now? (y/N) '
@@ -51,12 +49,24 @@ def check_apply(args: dict, prompt: bool = True):
       if answer.lower() in [ 'n', '' ]:
         return 0
       else:
-        print('Applying idstools state...')
-        return subprocess.run(cmd_arr)
+        print('Applying changes:')
+        return apply()
     else:
       return 0
 
 
+def apply():
+  salt_cmd = ['salt-call', 'state.apply', '-l', 'quiet', 'idstools.sync_files', 'queue=True']
+  update_cmd = ['so-rule-update']
+  print('Syncing config files...')
+  cmd = subprocess.run(salt_cmd, stdout=subprocess.DEVNULL)
+  if cmd.returncode == 0:
+    print('Updating rules...')
+    return subprocess.run(update_cmd).returncode
+  else:
+    return cmd.returncode
+
+
 def find_minion_pillar() -> str:
   regex = '^.*_(manager|managersearch|standalone|import|eval)\.sls$'
   
@@ -442,10 +452,7 @@ def main():
         modify.print_help()
     sys.exit(0)
   
-  if isinstance(exit_code, subprocess.CompletedProcess):
-    sys.exit(exit_code.returncode)
-  else:
-    sys.exit(exit_code)
+  sys.exit(exit_code)
 
 
 if __name__ == '__main__':

salt/common/tools/sbin/so-sensor-clean

@@ -115,8 +115,8 @@ clean() {
 }
 
 # Check to see if we are already running
-IS_RUNNING=$(ps aux | grep "so-sensor-clean" | grep -v grep | wc -l)
-[ "$IS_RUNNING" -gt 2 ] && echo "$(date) - $IS_RUNNING sensor clean script processes running...exiting." >>$LOG && exit 0
+IS_RUNNING=$(ps aux | pgrep -f "so-sensor-clean" | wc -l)
+[ "$IS_RUNNING" -gt 3 ] && echo "$(date) - $IS_RUNNING sensor clean script processes running...exiting." >>$LOG && exit 0
 
 if [ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ]; then
 	while [ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ]; do

salt/common/tools/sbin/so-ssh-harden

@@ -4,90 +4,184 @@
 
 if [[ $1 =~ ^(-q|--quiet) ]]; then
 	quiet=true
+elif [[ $1 =~ ^(-v|--verbose) ]]; then
+	verbose=true
 fi
 
+sshd_config=/etc/ssh/sshd_config
+temp_config=/tmp/sshd_config
+
 before=
 after=
 reload_required=false
+change_header_printed=false
 
-print_sshd_t() {
+check_sshd_t() {
 	local string=$1
-	local state=$2
-	echo "${state}:"
 
 	local grep_out
 	grep_out=$(sshd -T | grep "^${string}")
 
-	if [[ $state == "Before" ]]; then
-		before=$grep_out
+	before=$grep_out
+}
+
+print_diff() {
+	local diff
+	diff=$(diff -dbB <(echo $before) <(echo $after) | awk 'NR>1')
+
+	if [[ -n $diff ]]; then
+		if [[ $change_header_printed == false ]]; then
+			printf '%s\n' '' "Changes" '-------' ''
+			change_header_printed=true
+		fi
+		echo -e "$diff\n"
+	fi
+}
+
+replace_or_add() {
+	local type=$1
+	local string=$2
+	if grep -q "$type" $temp_config; then
+		sed -i "/$type .*/d" $temp_config
+	fi
+	printf "%s\n\n" "$string" >> $temp_config
+	reload_required=true
+}
+
+test_config() {
+	local msg
+	msg=$(sshd -t -f $temp_config)
+	local ret=$?
+
+	if [[ -n $msg ]]; then
+		echo "Error found in temp sshd config:"
+		echo $msg
+	fi
+
+	return $ret
+}
+
+main() {
+	if ! [[ $quiet ]]; then echo "Copying current config to $temp_config"; fi
+	cp $sshd_config $temp_config
+
+	# Add newline to ssh for legibility
+	echo "" >> $temp_config
+
+	# Ciphers
+	check_sshd_t "ciphers"
+
+	local bad_ciphers=(
+		"3des-cbc"
+		"aes128-cbc"
+		"aes192-cbc"
+		"aes256-cbc"
+		"arcfour"
+		"arcfour128"
+		"arcfour256"
+		"blowfish-cbc"
+		"cast128-cbc"
+	)
+
+	local cipher_string=$before
+	for cipher in "${bad_ciphers[@]}"; do
+		cipher_string=$(echo "$cipher_string" | sed "s/${cipher}\(,\|\$\)//g" | sed 's/,$//')
+	done
+
+	after=$cipher_string
+
+	if [[ $verbose ]]; then print_diff; fi
+
+	if [[ $before != "$after" ]]; then
+		replace_or_add "ciphers" "$cipher_string" && test_config || exit 1
+	fi
+
+	# KexAlgorithms
+	check_sshd_t "kexalgorithms"
+
+	local bad_kexalgs=(
+		"diffie-hellman-group-exchange-sha1"
+		"diffie-hellman-group-exchange-sha256"
+		"diffie-hellman-group1-sha1"
+		"diffie-hellman-group14-sha1"
+		"ecdh-sha2-nistp256"
+		"ecdh-sha2-nistp521"
+		"ecdh-sha2-nistp384"
+	)
+
+	local kexalg_string=$before
+	for kexalg in "${bad_kexalgs[@]}"; do
+		kexalg_string=$(echo "$kexalg_string" | sed "s/${kexalg}\(,\|\$\)//g" | sed 's/,$//')
+	done
+
+	after=$kexalg_string
+
+	if [[ $verbose ]]; then print_diff; fi
+
+	if [[ $before != "$after" ]]; then
+		replace_or_add "kexalgorithms" "$kexalg_string" && test_config || exit 1
+	fi
+
+	# Macs
+	check_sshd_t "macs"
+	
+	local bad_macs=(
+		"hmac-sha2-512"
+		"umac-128@openssh.com"
+		"hmac-sha2-256"
+		"umac-64@openssh.com"
+		"hmac-sha1"
+		"hmac-sha1-etm@openssh.com"
+		"umac-64-etm@openssh.com"
+	)
+
+	local macs_string=$before
+	for mac in "${bad_macs[@]}"; do
+		macs_string=$(echo "$macs_string" | sed "s/${mac}\(,\|\$\)//g" | sed 's/,$//')
+	done
+
+	after=$macs_string
+
+	if [[ $verbose ]]; then print_diff; fi
+
+	if [[ $before != "$after" ]]; then
+		replace_or_add "macs" "$macs_string" && test_config || exit 1
+	fi
+
+	# HostKeyAlgorithms
+	check_sshd_t "hostkeyalgorithms"
+	
+	local optional_suffix_regex_hka="\(-cert-v01@openssh.com\)\?"
+	local bad_hostkeyalg_list=(
+		"ecdsa-sha2-nistp256"
+		"ecdsa-sha2-nistp384"
+		"ecdsa-sha2-nistp521"
+		"ssh-rsa"
+		"ssh-dss"
+	)
+
+	local hostkeyalg_string=$before
+	for alg in "${bad_hostkeyalg_list[@]}"; do
+		hostkeyalg_string=$(echo "$hostkeyalg_string" | sed "s/${alg}${optional_suffix_regex_hka}\(,\|\$\)//g" | sed 's/,$//')
+	done
+
+	after=$hostkeyalg_string
+	
+	if [[ $verbose ]]; then print_diff; fi
+	
+	if [[ $before != "$after" ]]; then
+		replace_or_add "hostkeyalgorithms" "$hostkeyalg_string" && test_config || exit 1
+	fi
+
+	if [[ $reload_required == true ]]; then
+		mv -f $temp_config $sshd_config
+		if ! [[ $quiet ]]; then echo "Reloading sshd to load config changes"; fi
+		systemctl reload sshd
+		echo "[ WARNING ] Any new ssh sessions will need to remove and reaccept the host key fingerprint for this server before reconnecting."
 	else
-		after=$grep_out
-	fi
-
-	echo $grep_out
-}
-
-print_msg() {
-	local msg=$1
-	if ! [[ $quiet ]]; then
-	printf "%s\n" \
-		"----" \
-		"$msg" \
-		"----" \
-		""
+		if ! [[ $quiet ]]; then echo "No changes made to temp file, cleaning up"; fi
+		rm -f $temp_config
 	fi
 }
 
-if ! [[ $quiet ]]; then print_sshd_t "ciphers" "Before"; fi
-sshd -T | grep "^ciphers" | sed -e "s/\(3des-cbc\|aes128-cbc\|aes192-cbc\|aes256-cbc\|arcfour\|arcfour128\|arcfour256\|blowfish-cbc\|cast128-cbc\|rijndael-cbc@lysator.liu.se\)\,\?//g" >> /etc/ssh/sshd_config
-if ! [[ $quiet ]]; then
-	print_sshd_t "ciphers" "After"
-	echo ""
-fi
-
-if [[ $before != $after ]]; then
-	reload_required=true
-fi
-
-if ! [[ $quiet ]]; then print_sshd_t "kexalgorithms" "Before"; fi
-sshd -T | grep "^kexalgorithms" | sed -e "s/\(diffie-hellman-group14-sha1\|ecdh-sha2-nistp256\|diffie-hellman-group-exchange-sha256\|diffie-hellman-group1-sha1\|diffie-hellman-group-exchange-sha1\|ecdh-sha2-nistp521\|ecdh-sha2-nistp384\)\,\?//g" >> /etc/ssh/sshd_config
-if ! [[ $quiet ]]; then
-	print_sshd_t "kexalgorithms" "After"
-	echo ""
-fi
-
-if [[ $before != $after ]]; then
-	reload_required=true
-fi
-
-if ! [[ $quiet ]]; then print_sshd_t "macs" "Before"; fi
-sshd -T | grep "^macs" | sed -e "s/\(hmac-sha2-512,\|umac-128@openssh.com,\|hmac-sha2-256,\|umac-64@openssh.com,\|hmac-sha1,\|hmac-sha1-etm@openssh.com,\|umac-64-etm@openssh.com,\|hmac-sha1\)//g" >> /etc/ssh/sshd_config
-if ! [[ $quiet ]]; then
-	print_sshd_t "macs" "After"
-	echo ""
-fi
-
-if [[ $before != $after ]]; then
-	reload_required=true
-fi
-
-if ! [[ $quiet ]]; then print_sshd_t "hostkeyalgorithms" "Before"; fi
-sshd -T | grep "^hostkeyalgorithms" | sed "s|ecdsa-sha2-nistp256,||g" | sed "s|ssh-rsa,||g" >> /etc/ssh/sshd_config
-if ! [[ $quiet ]]; then
-	print_sshd_t "hostkeyalgorithms" "After"
-	echo ""
-fi
-
-if [[ $before != $after ]]; then
-	reload_required=true
-fi
-
-if [[ $reload_required == true ]]; then
-	print_msg "Reloading sshd to load config changes..."
-	systemctl reload sshd
-fi
-
-{% if grains['os'] != 'CentOS' %}
-print_msg "[ WARNING ] Any new ssh sessions will need to remove and reaccept the ECDSA key for this server before reconnecting."
-{% endif %}
-
+main

salt/common/tools/sbin/so-zeek-stats

@@ -24,11 +24,11 @@ show_stats() {
   echo
   echo "Average throughput:"
   echo
-  docker exec so-zeek env -i PATH=/bin:/usr/bin:/sbin:/usr/sbin:/opt/bin:/usr/local/bin:/usr/local/sbin runuser -l zeek -c '/opt/zeek/bin/zeekctl capstats'
+  docker exec so-zeek env -i PATH=/bin:/usr/bin:/sbin:/usr/sbin:/opt/bin:/usr/local/bin:/usr/local/sbin /opt/zeek/bin/zeekctl capstats
   echo
   echo "Average packet loss:"
   echo
-  docker exec so-zeek env -i PATH=/bin:/usr/bin:/sbin:/usr/sbin:/opt/bin:/usr/local/bin:/usr/local/sbin runuser -l zeek -c '/opt/zeek/bin/zeekctl netstats'
+  docker exec so-zeek env -i PATH=/bin:/usr/bin:/sbin:/usr/sbin:/opt/bin:/usr/local/bin:/usr/local/sbin /opt/zeek/bin/zeekctl netstats
   echo
 }
 

salt/common/tools/sbin/soup

@@ -19,12 +19,11 @@
 
 UPDATE_DIR=/tmp/sogh/securityonion
 INSTALLEDVERSION=$(cat /etc/soversion)
+POSTVERSION=$INSTALLEDVERSION
 INSTALLEDSALTVERSION=$(salt --versions-report | grep Salt: | awk {'print $2'})
-DEFAULT_SALT_DIR=/opt/so/saltstack/default
 BATCHSIZE=5
 SOUP_LOG=/root/soup.log
-
-exec 3>&1 1>${SOUP_LOG} 2>&1
+WHATWOULDYOUSAYYAHDOHERE=soup
 
 add_common() {
   cp $UPDATE_DIR/salt/common/tools/sbin/so-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/
@@ -101,19 +100,6 @@ update_registry() {
   salt-call state.apply registry queue=True
 }
 
-check_airgap() {
-  # See if this is an airgap install
-  AIRGAP=$(cat /opt/so/saltstack/local/pillar/global.sls | grep airgap: | awk '{print $2}')
-  if [[ "$AIRGAP" == "True" ]]; then
-      is_airgap=0
-      UPDATE_DIR=/tmp/soagupdate/SecurityOnion
-      AGDOCKER=/tmp/soagupdate/docker
-      AGREPO=/tmp/soagupdate/Packages
-  else 
-      is_airgap=1
-  fi
-}
-
 check_sudoers() {
 	if grep -q "so-setup" /etc/sudoers; then
 		echo "There is an entry for so-setup in the sudoers file, this can be safely deleted using \"visudo\"."
@@ -174,6 +160,34 @@ check_log_size_limit() {
   fi
 }
 
+check_os_updates() {
+  # Check to see if there are OS updates
+  NEEDUPDATES="We have detected missing operating system (OS) updates. Do you want to install these OS updates now? This could take a while depending on the size of your grid and how many packages are missing, but it is recommended to keep your system updated."
+  if [[ $OS == 'ubuntu' ]]; then
+    OSUPDATES=$(apt list --upgradeable | grep -v "^Listing..." | grep -v "^docker-ce" | grep -v "^wazuh-" | grep -v "^salt-" | wc -l)
+  else
+    OSUPDATES=$(yum -q list updates | wc -l)
+  fi
+  if [[ "$OSUPDATES" -gt 0 ]]; then
+      echo $NEEDUPDATES
+      echo ""
+      read -p "Press U to update OS packages (recommended), C to continue without updates, or E to exit: " confirm
+
+      if [[ "$confirm" == [cC] ]]; then
+          echo "Continuing without updating packages"
+      elif [[ "$confirm" == [uU] ]]; then
+          echo "Applying Grid Updates"
+          salt \* -b 5 state.apply patch.os queue=True
+      else
+          echo "Exiting soup"
+          exit 0
+      fi
+  else
+      echo "Looks like you have an updated OS"
+  fi
+    
+}
+
 clean_dockers() {
   # Place Holder for cleaning up old docker images
   echo "Trying to clean up old dockers."
@@ -199,21 +213,11 @@ clone_to_tmp() {
   fi
 }
 
-copy_new_files() {
-  # Copy new files over to the salt dir
-  cd $UPDATE_DIR
-  rsync -a salt $DEFAULT_SALT_DIR/
-  rsync -a pillar $DEFAULT_SALT_DIR/
-  chown -R socore:socore $DEFAULT_SALT_DIR/
-  chmod 755 $DEFAULT_SALT_DIR/pillar/firewall/addfirewall.sh
-  cd /tmp
-}
-
 generate_and_clean_tarballs() {
   local new_version
   new_version=$(cat $UPDATE_DIR/VERSION)
   [ -d /opt/so/repo ] || mkdir -p /opt/so/repo
-  tar -czf "/opt/so/repo/$new_version.tar.gz" "$UPDATE_DIR"
+  tar -czf "/opt/so/repo/$new_version.tar.gz" -C "$UPDATE_DIR" .
   find "/opt/so/repo" -type f -not -name "$new_version.tar.gz" -exec rm -rf {} \;
 }
 
@@ -243,29 +247,53 @@ masterunlock() {
   fi
 }
 
-playbook() {
-  echo "Applying playbook settings"
-  if [[ "$INSTALLEDVERSION" =~ rc.1 ]]; then
-    salt-call state.apply playbook.OLD_db_init
-    rm -f /opt/so/rules/elastalert/playbook/*.yaml
-    so-playbook-ruleupdate >> /root/soup_playbook_rule_update.log 2>&1 &
-  fi
-  if [[ "$INSTALLEDVERSION" != 2.3.30 ]]; then
-    so-playbook-sigma-refresh >> /root/soup_playbook_sigma_refresh.log 2>&1 &
-  fi
+preupgrade_changes_2.3.50_repo() {
+    # We made repo changes in 2.3.50 and this prepares for that on upgrade
+    echo "Checking to see if 2.3.50 repo changes are needed."
+
+    [[ "$INSTALLEDVERSION" == 2.3.30 || "$INSTALLEDVERSION" == 2.3.40 ]] && up_2.3.3X_to_2.3.50_repo
 }
 
-pillar_changes() {
+preupgrade_changes() {
     # This function is to add any new pillar items if needed.
-    echo "Checking to see if pillar changes are needed."
-    
+    echo "Checking to see if changes are needed."
+
     [[ "$INSTALLEDVERSION" =~ rc.1 ]] && rc1_to_rc2
     [[ "$INSTALLEDVERSION" =~ rc.2 ]] && rc2_to_rc3
     [[ "$INSTALLEDVERSION" =~ rc.3 ]] && rc3_to_2.3.0
     [[ "$INSTALLEDVERSION" == 2.3.0 || "$INSTALLEDVERSION" == 2.3.1 || "$INSTALLEDVERSION" == 2.3.2 || "$INSTALLEDVERSION" == 2.3.10 ]] && up_2.3.0_to_2.3.20
     [[ "$INSTALLEDVERSION" == 2.3.20 || "$INSTALLEDVERSION" == 2.3.21 ]] && up_2.3.2X_to_2.3.30
+    [[ "$INSTALLEDVERSION" == 2.3.30 || "$INSTALLEDVERSION" == 2.3.40 ]] && up_2.3.3X_to_2.3.50
 }
 
+postupgrade_changes() {
+    # This function is to add any new pillar items if needed.
+    echo "Running post upgrade processes."
+    
+    [[ "$POSTVERSION" =~ rc.1 ]] && post_rc1_to_rc2
+    [[ "$POSTVERSION" == 2.3.20 || "$POSTVERSION" == 2.3.21 ]] && post_2.3.2X_to_2.3.30
+    [[ "$POSTVERSION" == 2.3.30 ]] && post_2.3.30_to_2.3.40
+}
+
+post_rc1_to_2.3.21() {
+  salt-call state.apply playbook.OLD_db_init
+  rm -f /opt/so/rules/elastalert/playbook/*.yaml
+  so-playbook-ruleupdate >> /root/soup_playbook_rule_update.log 2>&1 &
+  POSTVERSION=2.3.21
+}
+
+post_2.3.2X_to_2.3.30() {
+  so-playbook-sigma-refresh >> /root/soup_playbook_sigma_refresh.log 2>&1 &
+  POSTVERSION=2.3.30
+}
+
+post_2.3.30_to_2.3.40() {
+  so-playbook-sigma-refresh >> /root/soup_playbook_sigma_refresh.log 2>&1 &
+  so-kibana-space-defaults
+  POSTVERSION=2.3.40
+}
+
+
 rc1_to_rc2() {
 
   # Move the static file to global.sls
@@ -296,15 +324,14 @@ rc1_to_rc2() {
   done </tmp/nodes.txt
   # Add the nodes back using hostname
   while read p; do
-      local NAME=$(echo $p | awk '{print $1}')
-      local EHOSTNAME=$(echo $p | awk -F"_" '{print $1}')
-      local IP=$(echo $p | awk '{print $2}')
-      echo "Adding the new cross cluster config for $NAME"
-      curl -XPUT http://localhost:9200/_cluster/settings -H'Content-Type: application/json' -d '{"persistent": {"search": {"remote": {"'$NAME'": {"skip_unavailable": "true", "seeds": ["'$EHOSTNAME':9300"]}}}}}'
+    local NAME=$(echo $p | awk '{print $1}')
+    local EHOSTNAME=$(echo $p | awk -F"_" '{print $1}')
+    local IP=$(echo $p | awk '{print $2}')
+    echo "Adding the new cross cluster config for $NAME"
+    curl -XPUT http://localhost:9200/_cluster/settings -H'Content-Type: application/json' -d '{"persistent": {"search": {"remote": {"'$NAME'": {"skip_unavailable": "true", "seeds": ["'$EHOSTNAME':9300"]}}}}}'
   done </tmp/nodes.txt
 
   INSTALLEDVERSION=rc.2
-
 }
 
 rc2_to_rc3() {
@@ -334,10 +361,10 @@ rc3_to_2.3.0() {
   fi
 
   {
-      echo "redis_settings:"
-      echo "  redis_maxmemory: 827"
-      echo "playbook:"
-      echo "  api_key: de6639318502476f2fa5aa06f43f51fb389a3d7f" 
+    echo "redis_settings:"
+    echo "  redis_maxmemory: 827"
+    echo "playbook:"
+    echo "  api_key: de6639318502476f2fa5aa06f43f51fb389a3d7f" 
   } >> /opt/so/saltstack/local/pillar/global.sls
 
   sed -i 's/playbook:/playbook_db:/' /opt/so/saltstack/local/pillar/secrets.sls
@@ -385,7 +412,6 @@ up_2.3.0_to_2.3.20(){
   fi
 
   INSTALLEDVERSION=2.3.20
-
 }
 
 up_2.3.2X_to_2.3.30() {
@@ -395,11 +421,11 @@ up_2.3.2X_to_2.3.30() {
     sed -i -r "s/ (\{\{.*}})$/ '\1'/g" "$pillar"
   done
 
-# Change the IMAGEREPO
+  # Change the IMAGEREPO
   sed -i "/  imagerepo: 'securityonion'/c\  imagerepo: 'security-onion-solutions'" /opt/so/saltstack/local/pillar/global.sls
   sed -i "/  imagerepo: securityonion/c\  imagerepo: 'security-onion-solutions'" /opt/so/saltstack/local/pillar/global.sls 
 
-# Strelka rule repo pillar addition
+  # Strelka rule repo pillar addition
   if [ $is_airgap -eq 0 ]; then
       # Add manager as default Strelka YARA rule repo
       sed -i "/^strelka:/a \\  repos: \n    - https://$HOSTNAME/repo/rules/strelka" /opt/so/saltstack/local/pillar/global.sls;
@@ -408,18 +434,86 @@ up_2.3.2X_to_2.3.30() {
       sed -i "/^strelka:/a \\  repos: \n    - https://github.com/Neo23x0/signature-base" /opt/so/saltstack/local/pillar/global.sls;
   fi
   check_log_size_limit
+  INSTALLEDVERSION=2.3.30
 }
 
-space_check() {
-  # Check to see if there is enough space
+up_2.3.3X_to_2.3.50_repo() {
+  echo "Performing 2.3.50 repo actions."
+  if [[ "$OS" == "centos" ]]; then
+    # Import GPG Keys
+    gpg_rpm_import
+    echo "Disabling fastestmirror."
+    disable_fastestmirror
+    echo "Deleting unneeded repo files."
+    DELREPOS=('CentOS-Base' 'CentOS-CR' 'CentOS-Debuginfo' 'docker-ce' 'CentOS-fasttrack' 'CentOS-Media' 'CentOS-Sources' 'CentOS-Vault' 'CentOS-x86_64-kernel' 'epel' 'epel-testing' 'saltstack' 'wazuh')
+
+    for DELREPO in "${DELREPOS[@]}"; do
+      if [[ -f "/etc/yum.repos.d/$DELREPO.repo" ]]; then
+        echo "Deleting $DELREPO.repo"
+        rm -f "/etc/yum.repos.d/$DELREPO.repo"
+      fi
+    done
+    if [ $is_airgap -eq 1 ]; then
+      # Copy the new repo file if not airgap
+      cp $UPDATE_DIR/salt/repo/client/files/centos/securityonion.repo /etc/yum.repos.d/
+      yum clean all
+      yum repolist
+    fi
+  fi
+}
+
+up_2.3.3X_to_2.3.50() {
+  
+  cat <<EOF > /tmp/supersed.txt
+/so-zeek:/ {
+  p;
+  n;
+  /shards:/ {
+    p;
+    n;
+    /warm:/ {
+      p;
+      n;
+      /close:/ {
+        s/close: 365/close: 45/;
+        p;
+        n;
+        /delete:/ {
+          s/delete: 45/delete: 365/;
+          p;
+          d;
+        }
+      }
+    }
+  }
+}
+p;
+EOF
+  sed -n -i -f /tmp/supersed.txt /opt/so/saltstack/local/pillar/global.sls
+  rm /tmp/supersed.txt
+  INSTALLEDVERSION=2.3.50
+}
+
+verify_upgradespace() {
   CURRENTSPACE=$(df -BG / | grep -v Avail | awk '{print $4}' | sed 's/.$//')
   if [ "$CURRENTSPACE" -lt "10" ]; then
-      echo "You are low on disk space. Upgrade will try and clean up space.";
-      clean_dockers
+      echo "You are low on disk space."
+      return 1
   else
-      echo "Plenty of space for upgrading"
+      return 0
   fi
-  
+}
+
+upgrade_space() {
+  if ! verify_upgradespace; then
+    clean_dockers
+    if ! verify_upgradespace; then
+      echo "There is not enough space to perform the upgrade. Please free up space and try again"
+      exit 1
+    fi
+  else
+      echo "You have enough space for upgrade. Proceeding with soup."
+  fi  
 }
 
 thehive_maint() {
@@ -427,16 +521,16 @@ thehive_maint() {
   COUNT=0
   THEHIVE_CONNECTED="no"
   while [[ "$COUNT" -le 240 ]]; do
-      curl --output /dev/null --silent --head --fail -k "https://localhost/thehive/api/alert"
-          if [ $? -eq 0 ]; then
-              THEHIVE_CONNECTED="yes"
-              echo "connected!"
-              break
-          else
-              ((COUNT+=1))
-              sleep 1
-              echo -n "."
-          fi
+    curl --output /dev/null --silent --head --fail -k "https://localhost/thehive/api/alert"
+      if [ $? -eq 0 ]; then
+        THEHIVE_CONNECTED="yes"
+        echo "connected!"
+        break
+      else
+        ((COUNT+=1))
+        sleep 1
+        echo -n "."
+      fi
   done
   if [ "$THEHIVE_CONNECTED" == "yes" ]; then
     echo "Migrating thehive databases if needed."
@@ -467,87 +561,96 @@ update_version() {
   # Update the version to the latest
   echo "Updating the Security Onion version file."
   echo $NEWVERSION > /etc/soversion
+  echo $HOTFIXVERSION > /etc/sohotfix
   sed -i "/  soversion:/c\  soversion: $NEWVERSION" /opt/so/saltstack/local/pillar/global.sls
 }
 
 upgrade_check() {
-    # Let's make sure we actually need to update.
-    NEWVERSION=$(cat $UPDATE_DIR/VERSION)
-    if [ "$INSTALLEDVERSION" == "$NEWVERSION" ]; then
+  # Let's make sure we actually need to update.
+  NEWVERSION=$(cat $UPDATE_DIR/VERSION)
+  HOTFIXVERSION=$(cat $UPDATE_DIR/HOTFIX)
+  CURRENTHOTFIX=$(cat /etc/sohotfix 2>/dev/null)
+  if [ "$INSTALLEDVERSION" == "$NEWVERSION" ]; then
+    echo "Checking to see if there are hotfixes needed"
+    if [ "$HOTFIXVERSION" == "$CURRENTHOTFIX" ]; then
       echo "You are already running the latest version of Security Onion."
       exit 0
-    fi 
+    else
+      echo "We need to apply a hotfix"
+      is_hotfix=true
+    fi
+  else
+    is_hotfix=false
+  fi 
+
 }
 
 upgrade_check_salt() {
-    NEWSALTVERSION=$(grep version: $UPDATE_DIR/salt/salt/master.defaults.yaml | awk {'print $2'})
-    if [ "$INSTALLEDSALTVERSION" == "$NEWSALTVERSION" ]; then
-      echo "You are already running the correct version of Salt for Security Onion."
-    else
-      UPGRADESALT=1
-    fi
+  NEWSALTVERSION=$(grep version: $UPDATE_DIR/salt/salt/master.defaults.yaml | awk {'print $2'})
+  if [ "$INSTALLEDSALTVERSION" == "$NEWSALTVERSION" ]; then
+    echo "You are already running the correct version of Salt for Security Onion."
+  else
+    UPGRADESALT=1
+  fi
 }   
 upgrade_salt() {
-      SALTUPGRADED=True
-      echo "Performing upgrade of Salt from $INSTALLEDSALTVERSION to $NEWSALTVERSION."
-      echo ""
-      # If CentOS
-      if [ "$OS" == "centos" ]; then
-        echo "Removing yum versionlock for Salt."
-        echo ""
-        yum versionlock delete "salt-*"
-        echo "Updating Salt packages and restarting services."
-        echo ""
-	if [ $is_airgap -eq 0 ]; then
-          sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -r -F -M -x python3 stable "$NEWSALTVERSION"
-	else 
-          sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -F -M -x python3 stable "$NEWSALTVERSION"
-	fi
-        echo "Applying yum versionlock for Salt."
-        echo ""
-        yum versionlock add "salt-*"
-      # Else do Ubuntu things
-      elif [ "$OS" == "ubuntu" ]; then
-        echo "Removing apt hold for Salt."
-        echo ""
-        apt-mark unhold "salt-common"
-        apt-mark unhold "salt-master"
-        apt-mark unhold "salt-minion"
-        echo "Updating Salt packages and restarting services."
-        echo ""
-        sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -F -M -x python3 stable "$NEWSALTVERSION"
-        echo "Applying apt hold for Salt."
-        echo ""
-        apt-mark hold "salt-common"
-        apt-mark hold "salt-master"
-        apt-mark hold "salt-minion"
-      fi
+  SALTUPGRADED=True
+  echo "Performing upgrade of Salt from $INSTALLEDSALTVERSION to $NEWSALTVERSION."
+  echo ""
+  # If CentOS
+  if [[ $OS == 'centos' ]]; then
+    echo "Removing yum versionlock for Salt."
+    echo ""
+    yum versionlock delete "salt-*"
+    echo "Updating Salt packages and restarting services."
+    echo ""
+    sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -r -F -M -x python3 stable "$NEWSALTVERSION"
+    echo "Applying yum versionlock for Salt."
+    echo ""
+    yum versionlock add "salt-*"
+  # Else do Ubuntu things
+  elif [[ $OS == 'ubuntu' ]]; then
+    echo "Removing apt hold for Salt."
+    echo ""
+    apt-mark unhold "salt-common"
+    apt-mark unhold "salt-master"
+    apt-mark unhold "salt-minion"
+    echo "Updating Salt packages and restarting services."
+    echo ""
+    sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -F -M -x python3 stable "$NEWSALTVERSION"
+    echo "Applying apt hold for Salt."
+    echo ""
+    apt-mark hold "salt-common"
+    apt-mark hold "salt-master"
+    apt-mark hold "salt-minion"
+  fi
 }
 
 verify_latest_update_script() {
-    # Check to see if the update scripts match. If not run the new one.
-    CURRENTSOUP=$(md5sum /opt/so/saltstack/default/salt/common/tools/sbin/soup | awk '{print $1}')
-    GITSOUP=$(md5sum $UPDATE_DIR/salt/common/tools/sbin/soup | awk '{print $1}')
-    CURRENTCMN=$(md5sum /opt/so/saltstack/default/salt/common/tools/sbin/so-common | awk '{print $1}')
-    GITCMN=$(md5sum $UPDATE_DIR/salt/common/tools/sbin/so-common | awk '{print $1}')
-    CURRENTIMGCMN=$(md5sum /opt/so/saltstack/default/salt/common/tools/sbin/so-image-common | awk '{print $1}')
-    GITIMGCMN=$(md5sum $UPDATE_DIR/salt/common/tools/sbin/so-image-common | awk '{print $1}')
+  # Check to see if the update scripts match. If not run the new one.
+  CURRENTSOUP=$(md5sum /opt/so/saltstack/default/salt/common/tools/sbin/soup | awk '{print $1}')
+  GITSOUP=$(md5sum $UPDATE_DIR/salt/common/tools/sbin/soup | awk '{print $1}')
+  CURRENTCMN=$(md5sum /opt/so/saltstack/default/salt/common/tools/sbin/so-common | awk '{print $1}')
+  GITCMN=$(md5sum $UPDATE_DIR/salt/common/tools/sbin/so-common | awk '{print $1}')
+  CURRENTIMGCMN=$(md5sum /opt/so/saltstack/default/salt/common/tools/sbin/so-image-common | awk '{print $1}')
+  GITIMGCMN=$(md5sum $UPDATE_DIR/salt/common/tools/sbin/so-image-common | awk '{print $1}')
 
-    if [[ "$CURRENTSOUP" == "$GITSOUP" && "$CURRENTCMN" == "$GITCMN" && "$CURRENTIMGCMN" == "$GITIMGCMN" ]]; then
-      echo "This version of the soup script is up to date. Proceeding."
-    else
-      echo "You are not running the latest soup version. Updating soup and its components. Might take multiple runs to complete"
-      cp $UPDATE_DIR/salt/common/tools/sbin/soup $DEFAULT_SALT_DIR/salt/common/tools/sbin/
-      cp $UPDATE_DIR/salt/common/tools/sbin/so-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/
-      cp $UPDATE_DIR/salt/common/tools/sbin/so-image-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/
-      salt-call state.apply common queue=True
-      echo ""
-      echo "soup has been updated. Please run soup again."
-      exit 0
-    fi
+  if [[ "$CURRENTSOUP" == "$GITSOUP" && "$CURRENTCMN" == "$GITCMN" && "$CURRENTIMGCMN" == "$GITIMGCMN" ]]; then
+    echo "This version of the soup script is up to date. Proceeding."
+  else
+    echo "You are not running the latest soup version. Updating soup and its components. Might take multiple runs to complete"
+    cp $UPDATE_DIR/salt/common/tools/sbin/soup $DEFAULT_SALT_DIR/salt/common/tools/sbin/
+    cp $UPDATE_DIR/salt/common/tools/sbin/so-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/
+    cp $UPDATE_DIR/salt/common/tools/sbin/so-image-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/
+    salt-call state.apply common queue=True
+    echo ""
+    echo "soup has been updated. Please run soup again."
+    exit 0
+  fi
 }
 
 main () {
+echo "### Preparing soup at `date` ###"
 while getopts ":b" opt; do
   case "$opt" in
     b ) # process option b
@@ -557,9 +660,10 @@ while getopts ":b" opt; do
          echo "Batch size must be a number greater than 0."
          exit 1
        fi
-      ;;
-    \? ) echo "Usage: cmd [-b]"
-      ;;
+    ;;
+    \? ) 
+      echo "Usage: cmd [-b]"
+    ;;
   esac
 done
 
@@ -573,6 +677,8 @@ check_airgap
 echo "Found that Security Onion $INSTALLEDVERSION is currently installed."
 echo ""
 set_os
+set_palette
+check_elastic_license
 echo ""
 if [ $is_airgap -eq 0 ]; then
   # Let's mount the ISO since this is airgap
@@ -583,7 +689,7 @@ else
   rm -rf $UPDATE_DIR
   clone_to_tmp
 fi
-
+check_os_updates
 echo ""
 echo "Verifying we have the latest soup script."
 verify_latest_update_script
@@ -599,153 +705,190 @@ fi
 
 echo "Let's see if we need to update Security Onion."
 upgrade_check
-space_check
+upgrade_space
 
 echo "Checking for Salt Master and Minion updates."
 upgrade_check_salt
 
-echo ""
-echo "Performing upgrade from Security Onion $INSTALLEDVERSION to Security Onion $NEWVERSION."
-echo ""
-echo "Updating dockers to $NEWVERSION."
-if [ $is_airgap -eq 0 ]; then
-  airgap_update_dockers
-else
-  update_registry
-  update_docker_containers "soup"
-  FEATURESCHECK=$(lookup_pillar features elastic)
-  if [[ "$FEATURESCHECK" == "True" ]]; then 
-    TRUSTED_CONTAINERS=(
-      "so-elasticsearch"
-      "so-filebeat"
-      "so-kibana"
-      "so-logstash" 
-    )
-    update_docker_containers "features" "-features"
-  fi
-fi
-echo ""
-echo "Stopping Salt Minion service."
-systemctl stop salt-minion
-echo "Killing any remaining Salt Minion processes."
-pkill -9 -ef /usr/bin/salt-minion
-echo ""
-echo "Stopping Salt Master service."
-systemctl stop salt-master
-echo ""
 
-# Does salt need upgraded. If so update it.
-if [ "$UPGRADESALT" == "1" ]; then
-  echo "Upgrading Salt"
-  # Update the repo files so it can actually upgrade
+if [ "$is_hotfix" == "true" ]; then
+  echo "Applying $HOTFIXVERSION" 
+  copy_new_files
+  echo ""
+  update_version
+  salt-call state.highstate -l info queue=True
+  
+else
+  echo ""
+  echo "Performing upgrade from Security Onion $INSTALLEDVERSION to Security Onion $NEWVERSION."
+  echo ""
+
+  echo "Updating dockers to $NEWVERSION."
   if [ $is_airgap -eq 0 ]; then
+    airgap_update_dockers
     update_centos_repo
     yum clean all
+    check_os_updates
+  else
+    update_registry
+    update_docker_containers "soup"
   fi
-  upgrade_salt
-fi
 
-echo "Checking if Salt was upgraded."
-echo ""
-# Check that Salt was upgraded
-if [[ $(salt --versions-report | grep Salt: | awk {'print $2'}) != "$NEWSALTVERSION" ]]; then
-  echo "Salt upgrade failed. Check of indicators of failure in $SOUP_LOG."
-  echo "Once the issue is resolved, run soup again."
-  echo "Exiting."
   echo ""
-  exit 1
-else
-  echo "Salt upgrade success."
+  echo "Stopping Salt Minion service."
+  systemctl stop salt-minion
+  echo "Killing any remaining Salt Minion processes."
+  pkill -9 -ef /usr/bin/salt-minion
   echo ""
-fi
-
-echo "Making pillar changes."
-pillar_changes
-echo ""
-
-if [ $is_airgap -eq 0 ]; then
-  echo "Updating Rule Files to the Latest."
-  update_airgap_rules
-fi
-
-# Only update the repo if its airgap
-if [[ $is_airgap -eq 0 ]] && [[ "$UPGRADESALT" != "1" ]]; then
-update_centos_repo
-fi
-
-echo ""
-echo "Copying new Security Onion code from $UPDATE_DIR to $DEFAULT_SALT_DIR."
-copy_new_files
-echo ""
-update_version
-
-echo ""
-echo "Locking down Salt Master for upgrade"
-masterlock
-
-echo ""
-echo "Starting Salt Master service."
-systemctl start salt-master
-
-# Only regenerate osquery packages if Fleet is enabled
-FLEET_MANAGER=$(lookup_pillar fleet_manager)
-FLEET_NODE=$(lookup_pillar fleet_node)
-if [[ "$FLEET_MANAGER" == "True" || "$FLEET_NODE" == "True" ]]; then
+  echo "Stopping Salt Master service."
+  systemctl stop salt-master
   echo ""
-  echo "Regenerating Osquery Packages.... This will take several minutes."
-  salt-call state.apply fleet.event_gen-packages -l info queue=True
+
+  preupgrade_changes_2.3.50_repo
+
+  # Does salt need upgraded. If so update it.
+  if [ "$UPGRADESALT" == "1" ]; then
+    echo "Upgrading Salt"
+    # Update the repo files so it can actually upgrade
+    upgrade_salt
+  fi
+
+  echo "Checking if Salt was upgraded."
   echo ""
-fi
+  # Check that Salt was upgraded
+  SALTVERSIONPOSTUPGRADE=$(salt --versions-report | grep Salt: | awk {'print $2'})
+  if [[ "$SALTVERSIONPOSTUPGRADE" != "$NEWSALTVERSION" ]]; then
+    echo "Salt upgrade failed. Check of indicators of failure in $SOUP_LOG."
+    echo "Once the issue is resolved, run soup again."
+    echo "Exiting."
+    echo ""
+    exit 1
+  else
+    echo "Salt upgrade success."
+    echo ""
+  fi
 
-echo ""
-echo "Running a highstate to complete the Security Onion upgrade on this manager. This could take several minutes."
-salt-call state.highstate -l info queue=True
-echo ""
-echo "Upgrade from $INSTALLEDVERSION to $NEWVERSION complete."
-
-echo ""
-echo "Stopping Salt Master to remove ACL"
-systemctl stop salt-master
-
-masterunlock
-
-echo ""
-echo "Starting Salt Master service."
-systemctl start salt-master
-echo "Running a highstate. This could take several minutes."
-salt-call state.highstate -l info queue=True
-playbook
-unmount_update
-thehive_maint
-
-if [ "$UPGRADESALT" == "1" ]; then
+  preupgrade_changes
   echo ""
-  echo "Upgrading Salt on the remaining Security Onion nodes from $INSTALLEDSALTVERSION to $NEWSALTVERSION."
+
   if [ $is_airgap -eq 0 ]; then
-    salt -C 'not *_eval and not *_helixsensor and not *_manager and not *_managersearch and not *_standalone' cmd.run "yum clean all"
+    echo "Updating Rule Files to the Latest."
+    update_airgap_rules
   fi
-  salt -C 'not *_eval and not *_helixsensor and not *_manager and not *_managersearch and not *_standalone' -b $BATCHSIZE state.apply salt.minion queue=True
+
+  # Only update the repo if its airgap
+  if [[ $is_airgap -eq 0 ]] && [[ "$UPGRADESALT" != "1" ]]; then
+  update_centos_repo
+  fi
+
   echo ""
+  echo "Copying new Security Onion code from $UPDATE_DIR to $DEFAULT_SALT_DIR."
+  copy_new_files
+  echo ""
+  update_version
+
+  echo ""
+  echo "Locking down Salt Master for upgrade"
+  masterlock
+
+  echo ""
+  echo "Starting Salt Master service."
+  systemctl start salt-master
+
+  # Only regenerate osquery packages if Fleet is enabled
+  FLEET_MANAGER=$(lookup_pillar fleet_manager)
+  FLEET_NODE=$(lookup_pillar fleet_node)
+  if [[ "$FLEET_MANAGER" == "True" || "$FLEET_NODE" == "True" ]]; then
+    echo ""
+    echo "Regenerating Osquery Packages.... This will take several minutes."
+    salt-call state.apply fleet.event_gen-packages -l info queue=True
+    echo ""
+  fi
+
+  echo ""
+  echo "Running a highstate to complete the Security Onion upgrade on this manager. This could take several minutes."
+  salt-call state.highstate -l info queue=True
+  echo ""
+  echo "Upgrade from $INSTALLEDVERSION to $NEWVERSION complete."
+
+  echo ""
+  echo "Stopping Salt Master to remove ACL"
+  systemctl stop salt-master
+
+  masterunlock
+
+  echo ""
+  echo "Starting Salt Master service."
+  systemctl start salt-master
+  echo "Running a highstate. This could take several minutes."
+  salt-call state.highstate -l info queue=True
+  postupgrade_changes
+  unmount_update
+  thehive_maint
+
+  if [ "$UPGRADESALT" == "1" ]; then
+    if [ $is_airgap -eq 0 ]; then
+      echo ""
+      echo "Cleaning repos on remote Security Onion nodes."
+      salt -C 'not *_eval and not *_helixsensor and not *_manager and not *_managersearch and not *_standalone and G@os:CentOS' cmd.run "yum clean all"
+      echo ""
+    fi
+  fi
+
+  check_sudoers
+
+  if [[ -n $lsl_msg ]]; then
+    case $lsl_msg in
+      'distributed')
+        echo "[INFO] The value of log_size_limit in any heavy node minion pillars may be incorrect."
+        echo " -> We recommend checking and adjusting the values as necessary."
+        echo " -> Minion pillar directory: /opt/so/saltstack/local/pillar/minions/"
+      ;;
+      'single-node')
+        # We can assume the lsl_details array has been set if lsl_msg has this value
+        echo "[WARNING] The value of log_size_limit (${lsl_details[0]}) does not match the recommended value of ${lsl_details[1]}."
+        echo " -> We recommend checking and adjusting the value as necessary."
+        echo " -> File: /opt/so/saltstack/local/pillar/minions/${lsl_details[2]}.sls"
+      ;;
+    esac
+  fi
+
+  NUM_MINIONS=$(ls /opt/so/saltstack/local/pillar/minions/*_*.sls | wc -l)
+
+  if [ $NUM_MINIONS -gt 1 ]; then
+
+    cat << EOF
+    
+  
+  
+This appears to be a distributed deployment. Other nodes should update themselves at the next Salt highstate (typically within 15 minutes). Do not manually restart anything until you know that all the search/heavy nodes in your deployment are updated. This is especially important if you are using true clustering for Elasticsearch.
+
+Each minion is on a random 15 minute check-in period and things like network bandwidth can be a factor in how long the actual upgrade takes. If you have a heavy node on a slow link, it is going to take a while to get the containers to it. Depending on what changes happened between the versions, Elasticsearch might not be able to talk to said heavy node until the update is complete.
+
+If it looks like you’re missing data after the upgrade, please avoid restarting services and instead make sure at least one search node has completed its upgrade. The best way to do this is to run 'sudo salt-call state.highstate' from a search node and make sure there are no errors. Typically if it works on one node it will work on the rest. Forward nodes are less complex and will update as they check in so you can monitor those from the Grid section of SOC.
+
+For more information, please see https://docs.securityonion.net/en/2.3/soup.html#distributed-deployments.
+
+EOF
+
+  fi
 fi
 
-check_sudoers
-
-if [[ -n $lsl_msg ]]; then
-  case $lsl_msg in
-    'distributed')
-      echo "[INFO] The value of log_size_limit in any heavy node minion pillars may be incorrect."
-      echo " -> We recommend checking and adjusting the values as necessary."
-      echo " -> Minion pillar directory: /opt/so/saltstack/local/pillar/minions/"
-    ;;
-    'single-node')
-      # We can assume the lsl_details array has been set if lsl_msg has this value
-      echo "[WARNING] The value of log_size_limit (${lsl_details[0]}) does not match the recommended value of ${lsl_details[1]}."
-      echo " -> We recommend checking and adjusting the value as necessary."
-      echo " -> File: /opt/so/saltstack/local/pillar/minions/${lsl_details[2]}.sls"
-    ;;
-  esac
-fi
-
+echo "### soup has been served at `date` ###"
 }
 
-main "$@" | tee /dev/fd/3
+cat << EOF
+
+SOUP - Security Onion UPdater
+
+Please review the following for more information about the update process and recent updates:
+https://docs.securityonion.net/soup
+https://blog.securityonion.net
+
+Press Enter to continue or Ctrl-C to cancel.
+
+EOF
+
+read input
+
+main "$@" | tee -a $SOUP_LOG

salt/curator/files/bin/so-curator-closed-delete-delete

@@ -4,12 +4,11 @@
 {%- if grains['role'] in ['so-node', 'so-heavynode'] %}
   {%- set ELASTICSEARCH_HOST = salt['pillar.get']('elasticsearch:mainip', '') -%}  
   {%- set ELASTICSEARCH_PORT = salt['pillar.get']('elasticsearch:es_port', '') -%}
-  {%- set LOG_SIZE_LIMIT = salt['pillar.get']('elasticsearch:log_size_limit', '') -%}
 {%- elif grains['role'] in ['so-eval', 'so-managersearch', 'so-standalone'] %}
   {%- set ELASTICSEARCH_HOST = salt['pillar.get']('manager:mainip', '') -%}
   {%- set ELASTICSEARCH_PORT = salt['pillar.get']('manager:es_port', '') -%}
-  {%- set LOG_SIZE_LIMIT = salt['pillar.get']('manager:log_size_limit', '') -%}
 {%- endif -%}
+{%- set LOG_SIZE_LIMIT = salt['pillar.get']('elasticsearch:log_size_limit', '') -%}
 
 # Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
@@ -35,7 +34,7 @@ overlimit() {
 
 closedindices() {
 
-        INDICES=$(curl -s -k {% if grains['role'] in ['so-node','so-heavynode'] %}https://{% endif %}{{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/_cat/indices?h=index\&expand_wildcards=closed 2> /dev/null)
+        INDICES=$(curl -s -k https://{{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/_cat/indices?h=index\&expand_wildcards=closed 2> /dev/null)
         [ $? -eq 1 ] && return false 
         echo ${INDICES} | grep -q -E "(logstash-|so-)"
 }
@@ -50,12 +49,12 @@ while overlimit && closedindices; do
 	# First, get the list of closed indices using _cat/indices?h=index\&expand_wildcards=closed.
 	# Then, sort by date by telling sort to use hyphen as delimiter and then sort on the third field.
 	# Finally, select the first entry in that sorted list.
-	OLDEST_INDEX=$(curl -s -k {% if grains['role'] in ['so-node','so-heavynode'] %}https://{% endif %}{{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/_cat/indices?h=index\&expand_wildcards=closed | grep -E "(logstash-|so-)" | sort -t- -k3 | head -1)
+	OLDEST_INDEX=$(curl -s -k https://{{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/_cat/indices?h=index\&expand_wildcards=closed | grep -E "(logstash-|so-)" | sort -t- -k3 | head -1)
 	
 	# Now that we've determined OLDEST_INDEX, ask Elasticsearch to delete it.
-	curl -XDELETE -k {% if grains['role'] in ['so-node','so-heavynode'] %}https://{% endif %}{{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/${OLDEST_INDEX}
+	curl -XDELETE -k https://{{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/${OLDEST_INDEX}
 
 	# Finally, write a log entry that says we deleted it.
 	echo "$(date) - Used disk space exceeds LOG_SIZE_LIMIT ({{LOG_SIZE_LIMIT}} GB) - Index ${OLDEST_INDEX} deleted ..." >> ${LOG}
 
-done
+done

salt/curator/files/curator.yml

@@ -12,11 +12,11 @@ client:
     - {{elasticsearch}}
   port: 9200
   url_prefix:
-{% if grains['role'] in ['so-node', 'so-heavynode'] %}  use_ssl: True{% else %}  use_ssl: False{% endif %}
+  use_ssl: True
   certificate:
   client_cert:
   client_key:
-{% if grains['role'] in ['so-node', 'so-heavynode'] %}  ssl_no_validate: True{% else %}  ssl_no_validate: False{% endif %}
+  ssl_no_validate: True
   http_auth:
   timeout: 30
   master_only: False

salt/docker_clean/init.sls

@@ -1,86 +1,9 @@
 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% if sls in allowed_states %}
 
-{% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %}
-{% set MANAGER = salt['grains.get']('master') %}
-{% set OLDVERSIONS = ['2.0.0-rc.1','2.0.1-rc.1','2.0.2-rc.1','2.0.3-rc.1','2.1.0-rc.2','2.2.0-rc.3','2.3.0','2.3.1','2.3.2','2.3.10','2.3.20']%}
-
-{% for VERSION in OLDVERSIONS %}
-remove_images_{{ VERSION }}:
-  docker_image.absent:
-    - force: True
-    - images:
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-acng:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-thehive-cortex:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-curator:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-domainstats:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-elastalert:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-elasticsearch:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-filebeat:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-fleet:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-fleet-launcher:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-freqserver:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-grafana:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-idstools:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-influxdb:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-kibana:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-kratos:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-logstash:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-minio:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-mysql:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-nginx:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-pcaptools:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-playbook:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-redis:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-soc:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-soctopus:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-steno:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-strelka-frontend:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-strelka-manager:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-strelka-backend:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-strelka-filestream:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-suricata:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-telegraf:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-thehive:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-thehive-es:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-wazuh:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-zeek:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-acng:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-thehive-cortex:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-curator:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-domainstats:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-elastalert:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-elasticsearch:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-filebeat:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-fleet:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-fleet-launcher:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-freqserver:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-grafana:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-idstools:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-influxdb:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-kibana:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-kratos:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-logstash:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-minio:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-mysql:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-nginx:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-pcaptools:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-playbook:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-redis:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-soc:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-soctopus:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-steno:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-strelka-frontend:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-strelka-manager:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-strelka-backend:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-strelka-filestream:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-suricata:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-telegraf:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-thehive:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-thehive-es:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-wazuh:{{ VERSION }}'
-      - '{{ MANAGER }}:5000/securityonion/so-zeek:{{ VERSION }}'
-{% endfor %}
+prune_images:
+  cmd.run:
+    - name: so-docker-prune
 
 {% else %}
 

salt/elastalert/defaults.yaml

@@ -16,8 +16,8 @@ elastalert:
     #aws_region: us-east-1
     #profile: test
     #es_url_prefix: elasticsearch
-    #use_ssl: True
-    #verify_certs: True
+    use_ssl: true
+    verify_certs: false
     #es_send_get_body_as: GET
     #es_username: someusername
     #es_password: somepassword

salt/elastalert/files/modules/so/playbook-es.py

@@ -4,6 +4,9 @@ from time import gmtime, strftime
 import requests,json
 from elastalert.alerts import Alerter
 
+import urllib3
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
 class PlaybookESAlerter(Alerter):
     """
     Use matched data to create alerts in elasticsearch
@@ -14,10 +17,10 @@ class PlaybookESAlerter(Alerter):
     def alert(self, matches):
        for match in matches:
             today = strftime("%Y.%m.%d", gmtime())
-            timestamp = strftime("%Y-%m-%d"'T'"%H:%M:%S", gmtime())
+            timestamp = strftime("%Y-%m-%d"'T'"%H:%M:%S"'.000Z', gmtime())
             headers = {"Content-Type": "application/json"}
             payload = {"rule": { "name": self.rule['play_title'],"case_template": self.rule['play_id'],"uuid": self.rule['play_id'],"category": self.rule['rule.category']},"event":{ "severity": self.rule['event.severity'],"module": self.rule['event.module'],"dataset": self.rule['event.dataset'],"severity_label": self.rule['sigma_level']},"kibana_pivot": self.rule['kibana_pivot'],"soc_pivot": self.rule['soc_pivot'],"play_url": self.rule['play_url'],"sigma_level": self.rule['sigma_level'],"event_data": match, "@timestamp": timestamp}
-            url = f"http://{self.rule['elasticsearch_host']}/so-playbook-alerts-{today}/_doc/"
+            url = f"https://{self.rule['elasticsearch_host']}/so-playbook-alerts-{today}/_doc/"
             requests.post(url, data=json.dumps(payload), headers=headers, verify=False)
                             
     def get_info(self):

salt/elastalert/init.sls

@@ -104,8 +104,9 @@ elastaconf:
 wait_for_elasticsearch:
   module.run:
     - http.wait_for_successful_query:
-      - url: 'http://{{MANAGER}}:9200/_cat/indices/.kibana*'
+      - url: 'https://{{MANAGER}}:9200/_cat/indices/.kibana*'
       - wait_for: 180
+      - verify_ssl: False
 
 so-elastalert:
   docker_container.running:

salt/elasticsearch/files/elasticsearch.yml

@@ -1,6 +1,5 @@
 {%- set NODE_ROUTE_TYPE = salt['pillar.get']('elasticsearch:node_route_type', 'hot') %}
 {%- set NODEIP = salt['pillar.get']('elasticsearch:mainip') %}
-{%- set FEATURES = salt['pillar.get']('elastic:features', False) %}
 {%- set TRUECLUSTER = salt['pillar.get']('elasticsearch:true_cluster', False) %}
 {%- if TRUECLUSTER is sameas true %}
   {%- set ESCLUSTERNAME = salt['pillar.get']('elasticsearch:true_cluster_name') %}
@@ -10,12 +9,6 @@
 {%- set NODE_ROLES = salt['pillar.get']('elasticsearch:node_roles', ['data', 'ingest']) %}
 cluster.name: "{{ ESCLUSTERNAME }}"
 network.host: 0.0.0.0
-
-# minimum_master_nodes need to be explicitly set when bound on a public IP
-# set to 1 to allow single node clusters
-# Details: https://github.com/elastic/elasticsearch/pull/17288
-#discovery.zen.minimum_master_nodes: 1
-# This is a test -- if this is here, then the volume is mounted correctly.
 path.logs: /var/log/elasticsearch
 action.destructive_requires_name: true
 transport.bind_host: 0.0.0.0
@@ -25,27 +18,23 @@ cluster.routing.allocation.disk.threshold_enabled: true
 cluster.routing.allocation.disk.watermark.low: 95%
 cluster.routing.allocation.disk.watermark.high: 98%
 cluster.routing.allocation.disk.watermark.flood_stage: 98%
-{%- if FEATURES is sameas true %}
 xpack.ml.enabled: false
-#xpack.security.enabled: false
-#xpack.security.transport.ssl.enabled: true
-#xpack.security.transport.ssl.verification_mode: none
-#xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/elasticsearch.key
-#xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/elasticsearch.crt
-#xpack.security.transport.ssl.certificate_authorities: [ "/usr/share/elasticsearch/config/ca.crt" ]
-#{%- if grains['role'] in ['so-node','so-heavynode'] %}
-#xpack.security.http.ssl.enabled: true
-#xpack.security.http.ssl.client_authentication: none
-#xpack.security.http.ssl.key: /usr/share/elasticsearch/config/elasticsearch.key
-#xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/elasticsearch.crt
-#xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/ca.crt
-#{%- endif %}
-#xpack.security.authc:
-#  anonymous:
-#    username: anonymous_user
-#    roles: superuser
-#    authz_exception: true
-{%- endif %}
+xpack.security.enabled: true
+xpack.security.transport.ssl.enabled: true
+xpack.security.transport.ssl.verification_mode: none
+xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/elasticsearch.key
+xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/elasticsearch.crt
+xpack.security.transport.ssl.certificate_authorities: [ "/usr/share/elasticsearch/config/ca.crt" ]
+xpack.security.http.ssl.enabled: true
+xpack.security.http.ssl.client_authentication: none
+xpack.security.http.ssl.key: /usr/share/elasticsearch/config/elasticsearch.key
+xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/elasticsearch.crt
+xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/ca.crt
+xpack.security.authc:
+  anonymous:
+    username: anonymous_user
+    roles: superuser
+    authz_exception: true
 node.name: {{ grains.host }}
 script.max_compilations_rate: 1000/1m
 {%- if TRUECLUSTER is sameas true %}

salt/elasticsearch/files/ingest/common

@@ -32,8 +32,6 @@
     { "rename":          { "field": "category",            "target_field": "event.category",              "ignore_failure": true, "ignore_missing": true  } },
     { "rename":          { "field": "message2.community_id", "target_field": "network.community_id",  "ignore_failure": true,  "ignore_missing": true  } },
     { "lowercase":       { "field": "event.dataset", "ignore_failure": true,  "ignore_missing": true  } },
-    { "convert":         { "field": "destination.port", "type": "integer",  "ignore_failure": true,  "ignore_missing": true  } },
-    { "convert":         { "field": "source.port", "type": "integer",  "ignore_failure": true,  "ignore_missing": true  } },
     { "convert":         { "field": "log.id.uid", "type": "string",  "ignore_failure": true,  "ignore_missing": true  } },
     { "convert":         { "field": "agent.id", "type": "string",  "ignore_failure": true,  "ignore_missing": true  } },
     { "convert":         { "field": "event.severity", "type": "integer",  "ignore_failure": true,  "ignore_missing": true  } },

salt/elasticsearch/files/ingest/http.status

@@ -0,0 +1,70 @@
+{
+  "description" : "http.status",
+  "processors" : [
+	{ "set": { "if": "ctx.http.status_code == 100",    "field": "http.status_message", "value": "Continue" } },
+	{ "set": { "if": "ctx.http.status_code == 101",    "field": "http.status_message", "value": "Switching Protocols" } },
+	{ "set": { "if": "ctx.http.status_code == 102",    "field": "http.status_message", "value": "Processing" } },
+	{ "set": { "if": "ctx.http.status_code == 103",    "field": "http.status_message", "value": "Early Hints" } },
+	{ "set": { "if": "ctx.http.status_code == 200",    "field": "http.status_message", "value": "OK" } },
+	{ "set": { "if": "ctx.http.status_code == 201",    "field": "http.status_message", "value": "Created" } },
+	{ "set": { "if": "ctx.http.status_code == 202",    "field": "http.status_message", "value": "Accepted" } },
+	{ "set": { "if": "ctx.http.status_code == 203",    "field": "http.status_message", "value": "Non-Authoritative Information" } },
+	{ "set": { "if": "ctx.http.status_code == 204",    "field": "http.status_message", "value": "No Content" } },
+	{ "set": { "if": "ctx.http.status_code == 205",    "field": "http.status_message", "value": "Reset Content" } },
+	{ "set": { "if": "ctx.http.status_code == 206",    "field": "http.status_message", "value": "Partial Content" } },
+	{ "set": { "if": "ctx.http.status_code == 207",    "field": "http.status_message", "value": "Multi-Status" } },
+	{ "set": { "if": "ctx.http.status_code == 208",    "field": "http.status_message", "value": "Already Reported" } },
+	{ "set": { "if": "ctx.http.status_code == 226",    "field": "http.status_message", "value": "IM Used" } },
+	{ "set": { "if": "ctx.http.status_code == 300",    "field": "http.status_message", "value": "Multiple Choices" } },
+	{ "set": { "if": "ctx.http.status_code == 301",    "field": "http.status_message", "value": "Moved Permanently" } },
+	{ "set": { "if": "ctx.http.status_code == 302",    "field": "http.status_message", "value": "Found" } },
+	{ "set": { "if": "ctx.http.status_code == 303",    "field": "http.status_message", "value": "See Other" } },
+	{ "set": { "if": "ctx.http.status_code == 304",    "field": "http.status_message", "value": "Not Modified" } },
+	{ "set": { "if": "ctx.http.status_code == 305",    "field": "http.status_message", "value": "Use Proxy" } },
+	{ "set": { "if": "ctx.http.status_code == 306",    "field": "http.status_message", "value": "(Unused)" } },
+	{ "set": { "if": "ctx.http.status_code == 307",    "field": "http.status_message", "value": "Temporary Redirect" } },
+	{ "set": { "if": "ctx.http.status_code == 308",    "field": "http.status_message", "value": "Permanent Redirect" } },
+	{ "set": { "if": "ctx.http.status_code == 400",    "field": "http.status_message", "value": "Bad Request" } },
+	{ "set": { "if": "ctx.http.status_code == 401",    "field": "http.status_message", "value": "Unauthorized" } },
+	{ "set": { "if": "ctx.http.status_code == 402",    "field": "http.status_message", "value": "Payment Required" } },
+	{ "set": { "if": "ctx.http.status_code == 403",    "field": "http.status_message", "value": "Forbidden" } },
+	{ "set": { "if": "ctx.http.status_code == 404",    "field": "http.status_message", "value": "Not Found" } },
+	{ "set": { "if": "ctx.http.status_code == 405",    "field": "http.status_message", "value": "Method Not Allowed" } },
+	{ "set": { "if": "ctx.http.status_code == 406",    "field": "http.status_message", "value": "Not Acceptable" } },
+	{ "set": { "if": "ctx.http.status_code == 407",    "field": "http.status_message", "value": "Proxy Authentication Required" } },
+	{ "set": { "if": "ctx.http.status_code == 408",    "field": "http.status_message", "value": "Request Timeout" } },
+	{ "set": { "if": "ctx.http.status_code == 409",    "field": "http.status_message", "value": "Conflict" } },
+	{ "set": { "if": "ctx.http.status_code == 410",    "field": "http.status_message", "value": "Gone" } },
+	{ "set": { "if": "ctx.http.status_code == 411",    "field": "http.status_message", "value": "Length Required" } },
+	{ "set": { "if": "ctx.http.status_code == 412",    "field": "http.status_message", "value": "Precondition Failed" } },
+	{ "set": { "if": "ctx.http.status_code == 413",    "field": "http.status_message", "value": "Payload Too Large" } },
+	{ "set": { "if": "ctx.http.status_code == 414",    "field": "http.status_message", "value": "URI Too Long" } },
+	{ "set": { "if": "ctx.http.status_code == 415",    "field": "http.status_message", "value": "Unsupported Media Type" } },
+	{ "set": { "if": "ctx.http.status_code == 416",    "field": "http.status_message", "value": "Range Not Satisfiable" } },
+	{ "set": { "if": "ctx.http.status_code == 417",    "field": "http.status_message", "value": "Expectation Failed" } },
+	{ "set": { "if": "ctx.http.status_code == 421",    "field": "http.status_message", "value": "Misdirected Request" } },
+	{ "set": { "if": "ctx.http.status_code == 422",    "field": "http.status_message", "value": "Unprocessable Entity" } },
+	{ "set": { "if": "ctx.http.status_code == 423",    "field": "http.status_message", "value": "Locked" } },
+	{ "set": { "if": "ctx.http.status_code == 424",    "field": "http.status_message", "value": "Failed Dependency" } },
+	{ "set": { "if": "ctx.http.status_code == 425",    "field": "http.status_message", "value": "Too Early" } },
+	{ "set": { "if": "ctx.http.status_code == 426",    "field": "http.status_message", "value": "Upgrade Required" } },
+	{ "set": { "if": "ctx.http.status_code == 427",    "field": "http.status_message", "value": "Unassigned" } },
+	{ "set": { "if": "ctx.http.status_code == 428",    "field": "http.status_message", "value": "Precondition Required" } },
+	{ "set": { "if": "ctx.http.status_code == 429",    "field": "http.status_message", "value": "Too Many Requests" } },
+	{ "set": { "if": "ctx.http.status_code == 430",    "field": "http.status_message", "value": "Unassigned" } },
+	{ "set": { "if": "ctx.http.status_code == 431",    "field": "http.status_message", "value": "Request Header Fields Too Large" } },
+	{ "set": { "if": "ctx.http.status_code == 451",    "field": "http.status_message", "value": "Unavailable For Legal Reasons" } },
+	{ "set": { "if": "ctx.http.status_code == 500",    "field": "http.status_message", "value": "Internal Server Error" } },
+	{ "set": { "if": "ctx.http.status_code == 501",    "field": "http.status_message", "value": "Not Implemented" } },
+	{ "set": { "if": "ctx.http.status_code == 502",    "field": "http.status_message", "value": "Bad Gateway" } },
+	{ "set": { "if": "ctx.http.status_code == 503",    "field": "http.status_message", "value": "Service Unavailable" } },
+	{ "set": { "if": "ctx.http.status_code == 504",    "field": "http.status_message", "value": "Gateway Timeout" } },
+	{ "set": { "if": "ctx.http.status_code == 505",    "field": "http.status_message", "value": "HTTP Version Not Supported" } },
+	{ "set": { "if": "ctx.http.status_code == 506",    "field": "http.status_message", "value": "Variant Also Negotiates" } },
+	{ "set": { "if": "ctx.http.status_code == 507",    "field": "http.status_message", "value": "Insufficient Storage" } },
+	{ "set": { "if": "ctx.http.status_code == 508",    "field": "http.status_message", "value": "Loop Detected" } },
+	{ "set": { "if": "ctx.http.status_code == 509",    "field": "http.status_message", "value": "Unassigned" } },
+	{ "set": { "if": "ctx.http.status_code == 510",    "field": "http.status_message", "value": "Not Extended" } },
+	{ "set": { "if": "ctx.http.status_code == 511",    "field": "http.status_message", "value": "Network Authentication Required" } }
+  ]
+}

salt/elasticsearch/files/ingest/osquery.live_query

@@ -0,0 +1,16 @@
+{
+  "description" : "osquery live query",
+  "processors" : [
+    {
+      "script": {
+        "lang": "painless",
+        "source": "def dict = ['columns': new HashMap()]; for (entry in ctx['rows'].entrySet()) { dict['columns'][entry.getKey()] = entry.getValue(); } ctx['result'] = dict; "
+      }
+    },
+    { "remove":      { "field": [ "rows" ], "ignore_missing": true, "ignore_failure": true } },
+    { "rename":      { "field": "distributed_query_execution_id",  "target_field": "result.query_id",       "ignore_missing": true  } },   
+    { "rename":      { "field": "computer_name",  "target_field": "host.hostname",       "ignore_missing": true  } },        
+    { "pipeline":    { "name": "osquery.normalize" } },
+    { "pipeline":    { "name": "common" } }
+  ]
+}

salt/elasticsearch/files/ingest/osquery.normalize

@@ -0,0 +1,14 @@
+{
+  "description" : "osquery normalize",
+  "processors" : [
+     { "rename":      { "field": "result.columns.cmdline",               "target_field": "process.command_line",       "ignore_missing": true  } },    
+     { "rename":      { "field": "result.columns.cwd",               "target_field": "process.working_directory",       "ignore_missing": true  } },   
+     { "rename":      { "field": "result.columns.name",               "target_field": "process.name",       "ignore_missing": true  } },    
+     { "rename":      { "field": "result.columns.path",               "target_field": "process.executable",       "ignore_missing": true  } }, 
+     { "rename":      { "field": "result.columns.pid",               "target_field": "process.pid",       "ignore_missing": true  } },    
+     { "rename":      { "field": "result.columns.parent",               "target_field": "process.ppid",       "ignore_missing": true  } }, 
+     { "rename":      { "field": "result.columns.uid",               "target_field": "user.id",       "ignore_missing": true  } },   
+     { "rename":      { "field": "result.columns.username",           "target_field": "user.name",       "ignore_missing": true  } },    
+     { "rename":      { "field": "result.columns.gid",               "target_field": "group.id",       "ignore_missing": true  } }     
+  ]
+}

salt/elasticsearch/files/ingest/osquery.query_result

@@ -1,24 +1,20 @@
 {
   "description" : "osquery",
   "processors" : [
-    { "json":        { "field": "message",                                            "target_field": "message2",             "ignore_failure": true  } },
-    { "gsub":        { "field": "message2.columns.data",                              "pattern": "\\\\xC2\\\\xAE",             "replacement": "", "ignore_missing": true  } },
-    { "rename":      { "if": "ctx.message2.columns?.eventid != null", "field": "message2.columns",  "target_field": "winlog",       "ignore_missing": true  } },
+    { "json":        { "field": "message",                                            "target_field": "result",             "ignore_failure": true  } },
+    { "gsub":        { "field": "result.columns.data",                              "pattern": "\\\\xC2\\\\xAE",             "replacement": "", "ignore_missing": true  } },
+    { "rename":      { "if": "ctx.result.columns?.eventid != null", "field": "result.columns",  "target_field": "winlog",       "ignore_missing": true  } },
     { "json":        { "field": "winlog.data",    "target_field": "unparsed",      "ignore_failure": true} },
     { "set":         { "if": "!(ctx.unparsed?.EventData instanceof Map)", "field": "error.eventdata_parsing", "value": true, "ignore_failure": true  } },     
     { "rename":      { "if": "!(ctx.error?.eventdata_parsing == true)", "field": "unparsed.EventData", "target_field": "winlog.event_data",       "ignore_missing": true, "ignore_failure": true  } },   
     { "rename":      { "field": "winlog.source",               "target_field": "winlog.channel",       "ignore_missing": true  } },    
     { "rename":      { "field": "winlog.eventid",               "target_field": "winlog.event_id",       "ignore_missing": true  } },   
+    { "rename":      { "field": "winlog.datetime",      "target_field": "winlog.systemTime",     "ignore_missing": true } },
     { "pipeline":    { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational'",   "name": "sysmon"  }  },
-    { "pipeline":    { "if": "ctx.winlog?.channel != 'Microsoft-Windows-Sysmon/Operational'",  "name":"win.eventlogs" }  },
-    {
-      "script": {
-        "lang": "painless",
-        "source": "def dict = ['result': new HashMap()]; for (entry in ctx['message2'].entrySet()) { dict['result'][entry.getKey()] = entry.getValue(); } ctx['osquery'] = dict; "
-      }
-    },
+    { "pipeline":    { "if": "ctx.winlog?.channel != 'Microsoft-Windows-Sysmon/Operational' && ctx.containsKey('winlog')",  "name":"win.eventlogs" }  },
     { "set":           { "field": "event.module", "value": "osquery", "override": false }  },
-    { "set":           { "field": "event.dataset", "value": "{{osquery.result.name}}", "override": false}  },
+    { "set":           { "field": "event.dataset", "value": "{{result.name}}", "override": false}  },
+    { "pipeline":    { "if": "!(ctx.containsKey('winlog'))",  "name": "osquery.normalize" } },
     { "pipeline":    { "name": "common" } }
   ]
 }

salt/elasticsearch/files/ingest/suricata.dhcp

@@ -1,13 +1,14 @@
 {
   "description" : "suricata.dhcp",
   "processors" : [
-    { "rename": 	{ "field": "message2.proto", 		"target_field": "network.transport",		"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.app_proto", 		"target_field": "network.protocol",		"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.dhcp.assigned_ip", 		"target_field": "dhcp.assigned_ip",		"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.dhcp.client_mac", 		"target_field": "host.mac",		"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.dhcp.dhcp_type", 		"target_field": "dhcp.message_types",		"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.dhcp.assigned_ip", 		"target_field": "dhcp.assigned_ip",		"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.dhcp.type", 		"target_field": "dhcp.type",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.proto", 		"target_field": "network.transport",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.app_proto", 	"target_field": "network.protocol",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.dhcp.assigned_ip",	"target_field": "dhcp.assigned_ip",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.dhcp.client_ip", 	"target_field": "client.address",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.dhcp.client_mac", 	"target_field": "host.mac",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.dhcp.dhcp_type", 	"target_field": "dhcp.message_types",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.dhcp.hostname", 	"target_field": "host.hostname",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.dhcp.type", 	"target_field": "dhcp.type",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.dhcp.id", 		"target_field": "dhcp.id",		"ignore_missing": true 	} },
     { "pipeline": { "name": "common" } }
   ]

salt/elasticsearch/files/ingest/suricata.http

@@ -1,17 +1,18 @@
 {
   "description" : "suricata.http",
   "processors" : [
-    { "rename": 	{ "field": "message2.proto", 		"target_field": "network.transport",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.proto", 			"target_field": "network.transport",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.app_proto", 		"target_field": "network.protocol",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.http.hostname", 		"target_field": "http.virtual_host",		"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.http.http_user_agent", 		"target_field": "http.useragent",		"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.http.url", 		"target_field": "http.uri",		"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.http.http_content_type", 		"target_field": "file.resp_mime_types",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.http.http_user_agent", 	"target_field": "http.useragent",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.http.url", 		"target_field": "http.uri",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.http.http_content_type", 	"target_field": "file.resp_mime_types",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.http.http_refer", 		"target_field": "http.referrer",		"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.http.http_method", 		"target_field": "http.method",		"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.http.protocol", 		"target_field": "http.version",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.http.http_method", 	"target_field": "http.method",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.http.protocol", 		"target_field": "http.version",			"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.http.status", 		"target_field": "http.status_code",		"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.http.length", 		"target_field": "http.request.body.length",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.http.length", 		"target_field": "http.request.body.length",	"ignore_missing": true 	} },
+    { "pipeline": 	{ "if": "ctx.http?.status_code != null",	"name": "http.status"							} },
     { "pipeline": { "name": "common" } }
   ]
-}
+}

salt/elasticsearch/files/ingest/win.eventlogs

@@ -4,8 +4,8 @@
     { "set":           { "if": "ctx.winlog?.channel != null",   "field": "event.module", "value": "windows_eventlog", "override": false, "ignore_failure": true }  },
     { "set":           { "if": "ctx.winlog?.channel != null",   "field": "event.dataset", "value": "{{winlog.channel}}", "override": true }  },
     { "set":           { "if": "ctx.winlog?.computer_name != null",   "field": "observer.name", "value": "{{winlog.computer_name}}", "override": true }  },
-    { "rename":         { "if": "ctx.winlog?.systemTime != null", "field": "@timestamp",      "target_field": "ingest.timestamp",     "ignore_missing": true } },
-    { "set":           { "if": "ctx.winlog?.systemTime != null",   "field": "@timestamp", "value": "{{winlog.systemTime}}", "override": true }  },
+    { "rename":       { "if": "ctx.winlog?.systemTime != null", "field": "@timestamp",      "target_field": "event.ingested",     "ignore_missing": true } },
+    { "date":         { "if": "ctx.winlog?.systemTime != null", "field": "winlog.systemTime",      "formats": ["yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSS'Z'","yyyy-MM-dd'T'HH:mm:ss.SSSSSSS'Z'"] } },    
     { "set":           { "field": "event.code", "value": "{{winlog.event_id}}", "override": true }  },  
     { "set":           { "field": "event.category", "value": "host", "override": true }  },   
     { "rename": 	   { "field": "winlog.event_data.SubjectUserName", 			  "target_field": "user.name",		"ignore_failure": true,   "ignore_missing": true 	} },

salt/elasticsearch/files/so-elasticsearch-pipelines

@@ -27,11 +27,7 @@ echo -n "Waiting for ElasticSearch..."
 COUNT=0
 ELASTICSEARCH_CONNECTED="no"
 while [[ "$COUNT" -le 240 ]]; do
-    {% if grains['role'] in ['so-node','so-heavynode'] %}
-	curl ${ELASTICSEARCH_AUTH} -k --output /dev/null --silent --head --fail -L https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"
-	{% else %}
-	curl ${ELASTICSEARCH_AUTH} --output /dev/null --silent --head --fail -L http://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"
-	{% endif %}
+    curl ${ELASTICSEARCH_AUTH} -k --output /dev/null --silent --head --fail -L https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"
 	if [ $? -eq 0 ]; then
 		ELASTICSEARCH_CONNECTED="yes"
 		echo "connected!"
@@ -51,11 +47,7 @@ fi
 cd ${ELASTICSEARCH_INGEST_PIPELINES}
 
 echo "Loading pipelines..."
-{% if grains['role'] in ['so-node','so-heavynode'] %}
 for i in *; do echo $i; RESPONSE=$(curl ${ELASTICSEARCH_AUTH} -k -XPUT -L https://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_ingest/pipeline/$i -H 'Content-Type: application/json' -d@$i 2>/dev/null); echo $RESPONSE; if [[ "$RESPONSE" == *"error"* ]]; then RETURN_CODE=1; fi; done
-{% else %}
-for i in *; do echo $i; RESPONSE=$(curl ${ELASTICSEARCH_AUTH} -XPUT -L http://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_ingest/pipeline/$i -H 'Content-Type: application/json' -d@$i 2>/dev/null); echo $RESPONSE; if [[ "$RESPONSE" == *"error"* ]]; then RETURN_CODE=1; fi; done
-{% endif %}
 echo
 
 cd - >/dev/null

salt/elasticsearch/files/sotls.yml

@@ -1,17 +0,0 @@
-keystore.path: /usr/share/elasticsearch/config/sokeys
-keystore.password: changeit 
-keystore.algorithm: SunX509
-truststore.path: /etc/pki/java/cacerts
-truststore.password: changeit
-truststore.algorithm: PKIX
-protocols:
-- TLSv1.2
-ciphers:
-- TLS_RSA_WITH_AES_128_CBC_SHA256
-- TLS_RSA_WITH_AES_256_GCM_SHA384
-transport.encrypted: true
-{%- if grains['role'] in ['so-node','so-heavynode'] %}
-http.encrypted: true
-{%- else %}
-http.encrypted: false
-{%- endif %}

salt/elasticsearch/init.sls

@@ -18,17 +18,10 @@
 {% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %}
 {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %}
 {% set MANAGER = salt['grains.get']('master') %}
-{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
 {% set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%}
 {% set TRUECLUSTER = salt['pillar.get']('elasticsearch:true_cluster', False) %}
 {% set MANAGERIP = salt['pillar.get']('global:managerip') %}
 
-{% if FEATURES is sameas true %}
-  {% set FEATUREZ = "-features" %}
-{% else %}
-  {% set FEATUREZ = '' %}
-{% endif %}
-
 {% if grains['role'] in ['so-eval','so-managersearch', 'so-manager', 'so-standalone', 'so-import'] %}
   {% set esclustername = salt['pillar.get']('manager:esclustername') %}
   {% set esheap = salt['pillar.get']('manager:esheap') %}
@@ -147,14 +140,6 @@ esyml:
     - group: 939
     - template: jinja
 
-sotls:
-  file.managed:
-    - name: /opt/so/conf/elasticsearch/sotls.yml
-    - source: salt://elasticsearch/files/sotls.yml
-    - user: 930
-    - group: 939
-    - template: jinja
-
 #sync templates to /opt/so/conf/elasticsearch/templates
 {% for TEMPLATE in TEMPLATES %}
 es_template_{{TEMPLATE.split('.')[0] | replace("/","_") }}:
@@ -186,7 +171,7 @@ eslogdir:
 
 so-elasticsearch:
   docker_container.running:
-    - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-elasticsearch:{{ VERSION }}{{ FEATUREZ }}
+    - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-elasticsearch:{{ VERSION }}
     - hostname: elasticsearch
     - name: so-elasticsearch
     - user: elasticsearch
@@ -206,7 +191,7 @@ so-elasticsearch:
       {% if TRUECLUSTER is sameas false or (TRUECLUSTER is sameas true and not salt['pillar.get']('nodestab', {})) %}
       - discovery.type=single-node
       {% endif %}
-      - ES_JAVA_OPTS=-Xms{{ esheap }} -Xmx{{ esheap }}
+      - ES_JAVA_OPTS=-Xms{{ esheap }} -Xmx{{ esheap }} -Des.transport.cname_in_publish_address=true
       ulimits:
       - memlock=-1:-1
       - nofile=65536:65536
@@ -228,7 +213,6 @@ so-elasticsearch:
       - /etc/pki/elasticsearch.crt:/usr/share/elasticsearch/config/elasticsearch.crt:ro
       - /etc/pki/elasticsearch.key:/usr/share/elasticsearch/config/elasticsearch.key:ro
       - /etc/pki/elasticsearch.p12:/usr/share/elasticsearch/config/elasticsearch.p12:ro
-      - /opt/so/conf/elasticsearch/sotls.yml:/usr/share/elasticsearch/config/sotls.yml:ro
     - watch:
       - file: cacertz
       - file: esyml

salt/elasticsearch/templates/so/so-common-template.json

@@ -51,16 +51,29 @@
             "match_mapping_type": "string",
             "path_match": "*.ip",
             "mapping": {
-              "type": "ip"
+              "type": "ip",
+              "fields" : {
+                "keyword" : {
+                  "ignore_above" : 45,
+                  "type" : "keyword"
+                }
+              }
+
             }
           }
        },
        {
           "port": {
-            "match_mapping_type": "string",
             "path_match": "*.port",
             "mapping": {
-              "type": "integer"
+              "type": "integer",
+              "fields" : {
+                "keyword" : {
+                  "ignore_above" : 6,
+                  "type" : "keyword"
+                }
+              }
+
             }
           }
        },
@@ -254,9 +267,14 @@
         },
         "ingest":{
           "type":"object",
-          "dynamic": true
+          "dynamic": true,
+          "properties":{
+            "timestamp":{
+              "type":"date"
+            }
+          }
         },
-        "intel":{
+	"intel":{
           "type":"object",
           "dynamic": true,
           "properties":{
@@ -365,6 +383,10 @@
 	      "request":{
           "type":"object",
           "dynamic": true
+        },
+	      "result":{
+          "type":"object",
+          "dynamic": true
         },
         "rfb":{
           "type":"object",

salt/filebeat/etc/filebeat.yml

@@ -260,7 +260,8 @@ output.{{ type }}:
   {%- if grains['role'] in ["so-eval", "so-import"] %}
 output.elasticsearch:
   enabled: true
-  hosts: ["{{ MANAGER }}:9200"]
+  hosts: ["https://{{ MANAGER }}:9200"]
+  ssl.certificate_authorities: ["/usr/share/filebeat/intraca.crt"]
   pipelines:
     - pipeline: "%{[module]}.%{[dataset]}"
   indices:
@@ -492,12 +493,13 @@ setup.template.enabled: false
 # append ?pretty to the URL.
 
 # Defines if the HTTP endpoint is enabled.
-#http.enabled: false
+http.enabled: true
 
 # The HTTP endpoint will bind to this hostname or IP address. It is recommended to use only localhost.
-#http.host: localhost
+http.host: 0.0.0.0
 
 # Port on which the HTTP endpoint will bind. Default is 5066.
+http.port: 5066
 
 queue.mem.events: {{ FBMEMEVENTS }}
 queue.mem.flush.min_events: {{ FBMEMFLUSHMINEVENTS }}

salt/filebeat/init.sls

@@ -13,7 +13,6 @@
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% if sls in allowed_states %}
-
 {% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %}
 {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %}
 {% set LOCALHOSTNAME = salt['grains.get']('host') %}
@@ -21,12 +20,6 @@
 {% set LOCALHOSTIP = salt['grains.get']('ip_interfaces').get(MAININT)[0] %}
 {% set MANAGER = salt['grains.get']('master') %}
 {% set MANAGERIP = salt['pillar.get']('global:managerip', '') %}
-{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
-{%- if FEATURES is sameas true %}
-  {% set FEATURES = "-features" %}
-{% else %}
-  {% set FEATURES = '' %}
-{% endif %}
 filebeatetcdir:
   file.directory:
     - name: /opt/so/conf/filebeat/etc
@@ -64,7 +57,7 @@ filebeatconfsync:
         OUTPUT: {{ salt['pillar.get']('filebeat:config:output', {}) }}
 so-filebeat:
   docker_container.running:
-    - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-filebeat:{{ VERSION }}{{ FEATURES }}
+    - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-filebeat:{{ VERSION }}
     - hostname: so-filebeat
     - user: root
     - extra_hosts: {{ MANAGER }}:{{ MANAGERIP }},{{ LOCALHOSTNAME }}:{{ LOCALHOSTIP }}
@@ -81,6 +74,7 @@ so-filebeat:
     - port_bindings:
         - 0.0.0.0:514:514/udp
         - 0.0.0.0:514:514/tcp
+        - 0.0.0.0:5066:5066/tcp
     - watch:
       - file: /opt/so/conf/filebeat/etc/filebeat.yml
 

salt/firewall/init.sls

@@ -26,15 +26,6 @@ iptables_fix_fwd:
     - position: 1
     - target: DOCKER-USER
 
-# Allow related/established sessions
-iptables_allow_established:
-  iptables.append:
-    - table: filter
-    - chain: INPUT
-    - jump: ACCEPT
-    - match: conntrack
-    - ctstate: 'RELATED,ESTABLISHED'
-
 # I like pings
 iptables_allow_pings:
   iptables.append:
@@ -77,17 +68,6 @@ enable_docker_user_fw_policy:
     - out-interface: docker0
     - position: 1
 
-enable_docker_user_established:
-  iptables.insert:
-    - table: filter
-    - chain: DOCKER-USER
-    - jump: ACCEPT
-    - in-interface: '!docker0'
-    - out-interface: docker0
-    - position: 1
-    - match: conntrack
-    - ctstate: 'RELATED,ESTABLISHED'
-
 {% set count = namespace(value=0) %}
 {% for chain, hg in assigned_hostgroups.chain.items() %}
   {% for hostgroup, portgroups in assigned_hostgroups.chain[chain].hostgroups.items() %}
@@ -120,6 +100,27 @@ enable_docker_user_established:
   {% endfor %}
 {% endfor %}
 
+# Allow related/established sessions
+iptables_allow_established:
+  iptables.insert:
+    - table: filter
+    - chain: INPUT
+    - jump: ACCEPT
+    - position: 1
+    - match: conntrack
+    - ctstate: 'RELATED,ESTABLISHED'
+
+enable_docker_user_established:
+  iptables.insert:
+    - table: filter
+    - chain: DOCKER-USER
+    - jump: ACCEPT
+    - in-interface: '!docker0'
+    - out-interface: docker0
+    - position: 1
+    - match: conntrack
+    - ctstate: 'RELATED,ESTABLISHED'
+
 # Block icmp timestamp response
 block_icmp_timestamp_reply:
   iptables.append:

salt/firewall/map.jinja

@@ -18,14 +18,18 @@
 
 {# This block translate the portgroups defined in the pillar to what is defined my portgroups.yaml and portgroups.local.yaml #}
 {% if salt['pillar.get']('firewall:assigned_hostgroups:chain') %}
+  {% set translated_pillar_assigned_hostgroups = {'chain': {}} %}
 
   {% for chain, hg in salt['pillar.get']('firewall:assigned_hostgroups:chain').items() %}
     {% for pillar_hostgroup, pillar_portgroups in salt['pillar.get']('firewall:assigned_hostgroups:chain')[chain].hostgroups.items() %}
-      {% do translated_pillar_assigned_hostgroups.update({"chain": {chain: {"hostgroups": {pillar_hostgroup: {"portgroups": []}}}}}) %}
+      {% if translated_pillar_assigned_hostgroups.chain[chain] is defined %}
+        {% do translated_pillar_assigned_hostgroups.chain[chain].hostgroups.update({pillar_hostgroup: {"portgroups": []}}) %}
+      {% else %}
+        {% do translated_pillar_assigned_hostgroups.chain.update({chain: {"hostgroups": {pillar_hostgroup: {"portgroups": []}}}}) %}
+      {% endif %}
       {% for pillar_portgroup in pillar_portgroups.portgroups %}
         {% set pillar_portgroup = pillar_portgroup.split('.') | last %}
         {% do translated_pillar_assigned_hostgroups.chain[chain].hostgroups[pillar_hostgroup].portgroups.append(defined_portgroups[pillar_portgroup]) %}
-
       {% endfor %}
     {% endfor %}
   {% endfor %}
@@ -39,7 +43,6 @@
   {% set assigned_hostgroups = default_assigned_hostgroups.role[role] %}
 {% endif %}
 
-
 {% if translated_pillar_assigned_hostgroups %}
   {% do salt['defaults.merge'](assigned_hostgroups, translated_pillar_assigned_hostgroups, merge_lists=True, in_place=True) %}
 {% endif %}

salt/firewall/portgroups.yaml

@@ -18,6 +18,9 @@ firewall:
       beats_5644:
         tcp:
           - 5644
+      beats_5066:
+        tcp:
+          - 5066
       cortex:
         tcp:
           - 9001

salt/grafana/dashboards/manager/manager.json

@@ -4322,139 +4322,6 @@
         "align": false,
         "alignLevel": null
       }
-    },
-    {
-      "aliasColors": {},
-      "bars": false,
-      "dashLength": 10,
-      "dashes": false,
-      "datasource": "InfluxDB",
-      "fieldConfig": {
-        "defaults": {
-          "custom": {}
-        },
-        "overrides": []
-      },
-      "fill": 1,
-      "fillGradient": 0,
-      "gridPos": {
-        "h": 6,
-        "w": 8,
-        "x": 16,
-        "y": 31
-      },
-      "hiddenSeries": false,
-      "id": 76,
-      "legend": {
-        "avg": false,
-        "current": false,
-        "max": false,
-        "min": false,
-        "show": false,
-        "total": false,
-        "values": false
-      },
-      "lines": true,
-      "linewidth": 1,
-      "nullPointMode": "connected",
-      "options": {
-        "alertThreshold": true
-      },
-      "percentage": false,
-      "pluginVersion": "7.3.4",
-      "pointradius": 2,
-      "points": false,
-      "renderer": "flot",
-      "seriesOverrides": [],
-      "spaceLength": 10,
-      "stack": false,
-      "steppedLine": false,
-      "targets": [
-        {
-          "alias": "EPS",
-          "groupBy": [
-            {
-              "params": [
-                "$__interval"
-              ],
-              "type": "time"
-            },
-            {
-              "params": [
-                "null"
-              ],
-              "type": "fill"
-            }
-          ],
-          "measurement": "esteps",
-          "orderByTime": "ASC",
-          "policy": "default",
-          "queryType": "randomWalk",
-          "refId": "A",
-          "resultFormat": "time_series",
-          "select": [
-            [
-              {
-                "params": [
-                  "eps"
-                ],
-                "type": "field"
-              },
-              {
-                "params": [],
-                "type": "mean"
-              }
-            ]
-          ],
-          "tags": [
-            {
-              "key": "host",
-              "operator": "=",
-              "value": "{{ SERVERNAME }}"
-            }
-          ]
-        }
-      ],
-      "thresholds": [],
-      "timeFrom": null,
-      "timeRegions": [],
-      "timeShift": null,
-      "title": "{{ SERVERNAME }} - Estimated EPS",
-      "tooltip": {
-        "shared": true,
-        "sort": 0,
-        "value_type": "individual"
-      },
-      "type": "graph",
-      "xaxis": {
-        "buckets": null,
-        "mode": "time",
-        "name": null,
-        "show": true,
-        "values": []
-      },
-      "yaxes": [
-        {
-          "format": "short",
-          "label": "EPS",
-          "logBase": 1,
-          "max": null,
-          "min": null,
-          "show": true
-        },
-        {
-          "format": "short",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": null,
-          "show": false
-        }
-      ],
-      "yaxis": {
-        "align": false,
-        "alignLevel": null
-      }
     }
   ],
   "refresh": false,

salt/grafana/dashboards/managersearch/managersearch.json

@@ -5157,7 +5157,7 @@
               "type": "fill"
             }
           ],
-          "measurement": "esteps",
+          "measurement": "consumptioneps",
           "orderByTime": "ASC",
           "policy": "default",
           "queryType": "randomWalk",

salt/grafana/dashboards/standalone/standalone.json

@@ -5562,7 +5562,7 @@
               "type": "fill"
             }
           ],
-          "measurement": "esteps",
+          "measurement": "consumptioneps",
           "orderByTime": "ASC",
           "policy": "default",
           "queryType": "randomWalk",

salt/grafana/init.sls

@@ -11,7 +11,7 @@
 {% set GRAFANA_SETTINGS = salt['grains.filter_by'](default_settings, default='grafana', merge=salt['pillar.get']('grafana', {})) %}
 
 
-{% if grains['role'] in ['so-manager', 'so-managersearch', 'so-eval', 'so-standalone'] and GRAFANA == 1 %}
+{% if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone'] or (grains.role == 'so-eval' and GRAFANA == 1) %}
 
 # Grafana all the things
 grafanadir:

salt/idstools/init.sls

@@ -19,13 +19,12 @@
 {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %}
 {% set MANAGER = salt['grains.get']('master') %}
 {% set ENGINE = salt['pillar.get']('global:mdengine', '') %}
+{% set proxy = salt['pillar.get']('manager:proxy') %}
+
+include:
+  - idstools.sync_files
+
 # IDSTools Setup
-idstoolsdir:
-  file.directory:
-    - name: /opt/so/conf/idstools/etc
-    - user: 939
-    - group: 939
-    - makedirs: True
 
 idstoolslogdir:
   file.directory:
@@ -34,14 +33,6 @@ idstoolslogdir:
     - group: 939
     - makedirs: True
 
-idstoolsetcsync:
-  file.recurse:
-    - name: /opt/so/conf/idstools/etc
-    - source: salt://idstools/etc
-    - user: 939
-    - group: 939
-    - template: jinja
-
 so-ruleupdatecron:
   cron.present:
     - name: /usr/sbin/so-rule-update > /opt/so/log/idstools/download.log 2>&1
@@ -49,28 +40,17 @@ so-ruleupdatecron:
     - minute: '1'
     - hour: '7'
 
-rulesdir:
-  file.directory:
-    - name: /opt/so/rules/nids
-    - user: 939
-    - group: 939
-    - makedirs: True
-
-# Don't show changes because all.rules can be large
-synclocalnidsrules:
-  file.recurse:
-    - name: /opt/so/rules/nids/
-    - source: salt://idstools/
-    - user: 939
-    - group: 939
-    - show_changes: False
-    - include_pat: 'E@.rules'
-
 so-idstools:
   docker_container.running:
     - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-idstools:{{ VERSION }}
     - hostname: so-idstools
     - user: socore
+    {% if proxy %}
+    - environment:
+      - http_proxy={{ proxy }}
+      - https_proxy={{ proxy }}
+      - no_proxy={{ salt['pillar.get']('manager:no_proxy') }}
+    {% endif %}
     - binds:
       - /opt/so/conf/idstools/etc:/opt/so/idstools/etc:ro
       - /opt/so/rules/nids:/opt/so/rules/nids:rw

salt/idstools/sync_files.sls

@@ -0,0 +1,46 @@
+# Copyright 2014,2015,2016,2017,2018 Security Onion Solutions, LLC
+
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+idstoolsdir:
+  file.directory:
+    - name: /opt/so/conf/idstools/etc
+    - user: 939
+    - group: 939
+    - makedirs: True
+
+idstoolsetcsync:
+  file.recurse:
+    - name: /opt/so/conf/idstools/etc
+    - source: salt://idstools/etc
+    - user: 939
+    - group: 939
+    - template: jinja
+
+rulesdir:
+  file.directory:
+    - name: /opt/so/rules/nids
+    - user: 939
+    - group: 939
+    - makedirs: True
+
+# Don't show changes because all.rules can be large
+synclocalnidsrules:
+  file.recurse:
+    - name: /opt/so/rules/nids/
+    - source: salt://idstools/
+    - user: 939
+    - group: 939
+    - show_changes: False
+    - include_pat: 'E@.rules'

salt/influxdb/etc/influxdb.conf

@@ -233,7 +233,7 @@
   # enabled = true
 
   # Determines whether the Flux query endpoint is enabled.
-  # flux-enabled = false
+  flux-enabled = true
 
   # The bind address used by the HTTP service.
   # bind-address = ":8086"

salt/influxdb/init.sls

@@ -6,7 +6,7 @@
 {% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %}
 {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %}
 
-{% if grains['role'] in ['so-manager', 'so-managersearch', 'so-eval', 'so-standalone'] and GRAFANA == 1 %}
+{% if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone'] or (grains.role == 'so-eval' and GRAFANA == 1) %}
 
 # Influx DB
 influxconfdir:

salt/kibana/bin/keepkibanahappy.sh

@@ -1,53 +0,0 @@
-{%- set ES = salt['pillar.get']('manager:mainip', '') -%}
-# Wait for ElasticSearch to come up, so that we can query for version infromation
-echo -n "Waiting for ElasticSearch..."
-COUNT=0
-ELASTICSEARCH_CONNECTED="no"
-while [[ "$COUNT" -le 30 ]]; do
-  curl --output /dev/null --silent --head --fail -L http://{{ ES }}:9200
-  if [ $? -eq 0 ]; then
-    ELASTICSEARCH_CONNECTED="yes"
-    echo "connected!"
-    break
-  else
-    ((COUNT+=1))
-    sleep 1
-    echo -n "."
-  fi
-done
-if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then
-  echo
-  echo -e "Connection attempt timed out.  Unable to connect to ElasticSearch.  \nPlease try: \n  -checking log(s) in /var/log/elasticsearch/\n  -running 'sudo docker ps' \n  -running 'sudo so-elastic-restart'"
-  echo
-
-  exit
-fi
-
-# Make sure Kibana is running
-MAX_WAIT=240
-
-# Check to see if Kibana is available
-wait_step=0
-  until curl -s -XGET -L http://{{ ES }}:5601 > /dev/null ; do
-  wait_step=$(( ${wait_step} + 1 ))
-  echo "Waiting on Kibana...Attempt #$wait_step"
-	  if [ ${wait_step} -gt ${MAX_WAIT} ]; then
-			  echo "ERROR: Kibana not available for more than ${MAX_WAIT} seconds."
-			  exit 5
-	  fi
-		  sleep 1s;
-  done
-
-
-# Apply Kibana template
-  echo
-  echo "Applying Kibana template..."
-  curl -s -XPUT -L http://{{ ES }}:9200/_template/kibana \
-       -H 'Content-Type: application/json' \
-       -d'{"index_patterns" : ".kibana", "settings": { "number_of_shards" : 1, "number_of_replicas" : 0 }, "mappings" : { "search": {"properties": {"hits": {"type": "integer"}, "version": {"type": "integer"}}}}}'
-  echo
-
-  curl -s -XPUT -L "{{ ES }}:9200/.kibana/_settings" \
-       -H 'Content-Type: application/json' \
-       -d'{"index" : {"number_of_replicas" : 0}}'
-  echo

salt/kibana/bin/so-kibana-config-load

@@ -3,6 +3,8 @@
 # {%- set FLEET_NODE = salt['pillar.get']('global:fleet_node', False) -%}
 # {%- set MANAGER = salt['pillar.get']('global:url_base', '') %}
 
+. /usr/sbin/so-common
+
 # Copy template file
 cp /opt/so/conf/kibana/saved_objects.ndjson.template /opt/so/conf/kibana/saved_objects.ndjson
 
@@ -14,5 +16,11 @@ cp /opt/so/conf/kibana/saved_objects.ndjson.template /opt/so/conf/kibana/saved_o
 # SOCtopus and Manager
 sed -i "s/PLACEHOLDER/{{ MANAGER }}/g" /opt/so/conf/kibana/saved_objects.ndjson
 
+wait_for_web_response "http://localhost:5601/app/kibana" "Elastic"
+## This hackery will be removed if using Elastic Auth ##
+
+# Let's snag a cookie from Kibana
+THECOOKIE=$(curl -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}')
+
 # Load saved objects
-curl -X POST "localhost:5601/api/saved_objects/_import?overwrite=true" -H "kbn-xsrf: true" --form file=@/opt/so/conf/kibana/saved_objects.ndjson > /dev/null 2>&1
+curl -b "sid=$THECOOKIE" -L -X POST "localhost:5601/api/saved_objects/_import?overwrite=true" -H "kbn-xsrf: true" --form file=@/opt/so/conf/kibana/saved_objects.ndjson >> /opt/so/log/kibana/misc.log

salt/kibana/etc/kibana.yml

@@ -1,11 +1,11 @@
 ---
 # Default Kibana configuration from kibana-docker.
 {%- set ES = salt['pillar.get']('manager:mainip', '') -%}
-{%- set FEATURES = salt['pillar.get']('elastic:features', False) %}
 server.name: kibana
 server.host: "0"
 server.basePath: /kibana
-elasticsearch.hosts: [ "http://{{ ES }}:9200" ]
+elasticsearch.hosts: [ "https://{{ ES }}:9200" ]
+elasticsearch.ssl.verificationMode: none
 #kibana.index: ".kibana"
 #elasticsearch.username: elastic
 #elasticsearch.password: changeme
@@ -14,3 +14,7 @@ elasticsearch.requestTimeout: 90000
 logging.dest: /var/log/kibana/kibana.log
 telemetry.enabled: false
 security.showInsecureClusterWarning: false
+xpack.security.authc.providers:
+  anonymous.anonymous1:
+    order: 0
+    credentials: "elasticsearch_anonymous_user" 

salt/kibana/init.sls

@@ -4,12 +4,6 @@
 {% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %}
 {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %}
 {% set MANAGER = salt['grains.get']('master') %}
-{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
-{%- if FEATURES is sameas true %}
-  {% set FEATURES = "-features" %}
-{% else %}
-  {% set FEATURES = '' %}
-{% endif %}
 
 # Add ES Group
 kibanasearchgroup:
@@ -73,7 +67,7 @@ kibanabin:
 # Start the kibana docker
 so-kibana:
   docker_container.running:
-    - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-kibana:{{ VERSION }}{{ FEATURES }}
+    - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-kibana:{{ VERSION }}
     - hostname: kibana
     - user: kibana
     - environment:
@@ -100,21 +94,10 @@ kibanadashtemplate:
     - user: 932
     - group: 939
 
-wait_for_kibana:
-  module.run:
-    - http.wait_for_successful_query:
-      - url: "http://{{MANAGER}}:5601/api/saved_objects/_find?type=config"
-      - wait_for: 900
-    - onchanges:
-      - file: kibanadashtemplate
-
 so-kibana-config-load:
   cmd.run:
     - name: /usr/sbin/so-kibana-config-load
     - cwd: /opt/so
-    - onchanges:
-      - wait_for_kibana
-
 
 # Keep the setting correct
 #KibanaHappy:

salt/logstash/init.sls

@@ -19,13 +19,6 @@
 {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %}
 {% set MANAGER = salt['grains.get']('master') %}
 {% set MANAGERIP = salt['pillar.get']('global:managerip') %}
-{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
-
-{%- if FEATURES is sameas true %}
-  {% set FEATURES = "-features" %}
-{% else %}
-  {% set FEATURES = '' %}
-{% endif %}
 
 # Logstash Section - Decide which pillar to use
 {% set lsheap = salt['pillar.get']('logstash_settings:lsheap', '') %}
@@ -146,7 +139,7 @@ lslogdir:
 
 so-logstash:
   docker_container.running:
-    - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-logstash:{{ VERSION }}{{ FEATURES }}
+    - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-logstash:{{ VERSION }}
     - hostname: so-logstash
     - name: so-logstash
     - user: logstash

salt/logstash/pipelines/config/so/0008_input_fleet_livequery.conf.jinja

@@ -0,0 +1,19 @@
+{%- set MANAGER = salt['grains.get']('master') %}
+{%- set THREADS = salt['pillar.get']('logstash_settings:ls_input_threads', '') %}
+{% set BATCH = salt['pillar.get']('logstash_settings:ls_pipeline_batch_size', 125) %}
+
+input {
+	redis {
+		host => '{{ MANAGER }}'
+		port => 6379
+		data_type => 'pattern_channel'
+		key => 'results_*'
+		type => 'live_query'
+        add_field => {
+        "module" => "osquery"
+          "dataset" => "live_query"
+          }        
+		threads => {{ THREADS }}
+		batch_count => {{ BATCH }}
+	}
+}

salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja

@@ -3,7 +3,6 @@
 {%- else %}
 {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
 {%- endif %}
-{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
 output {
   if [module] =~ "zeek" and "import" not in [tags] {
     elasticsearch {
@@ -13,10 +12,8 @@ output {
       template_name => "so-zeek"
       template => "/templates/so-zeek-template.json"
       template_overwrite => true
-      {%- if grains['role'] in ['so-node','so-heavynode'] %}
       ssl => true
       ssl_certificate_verification => false
-      {%- endif %}
     }
   }
 }

salt/logstash/pipelines/config/so/9002_output_import.conf.jinja

@@ -3,7 +3,6 @@
 {%- else %}
 {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
 {%- endif %}
-{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
 output {
   if "import" in [tags] {
     elasticsearch {
@@ -13,10 +12,8 @@ output {
       template_name => "so-import"
       template => "/templates/so-import-template.json"
       template_overwrite => true
-      {%- if grains['role'] in ['so-node','so-heavynode'] %}
       ssl => true
       ssl_certificate_verification => false
-      {%- endif %}
     }
   }
 }

salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja

@@ -3,7 +3,6 @@
 {%- else %}
 {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
 {%- endif %}
-{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
 output {
   if [event_type] == "sflow" {
     elasticsearch {
@@ -12,10 +11,8 @@ output {
       template_name => "so-flow"
       template => "/templates/so-flow-template.json"
       template_overwrite => true
-      {%- if grains['role'] in ['so-node','so-heavynode'] %}
       ssl => true
       ssl_certificate_verification => false
-      {%- endif %}
     }
   }
 }

salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja

@@ -3,7 +3,6 @@
 {%- else %}
 {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
 {%- endif %}
-{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
 output {
   if [event_type] == "ids" and "import" not in [tags] {
     elasticsearch {
@@ -12,10 +11,8 @@ output {
       template_name => "so-ids"
       template => "/templates/so-ids-template.json"
       template_overwrite => true
-      {%- if grains['role'] in ['so-node','so-heavynode'] %}
       ssl => true
       ssl_certificate_verification => false
-      {%- endif %}
     }
   }
 }

salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja

@@ -3,7 +3,6 @@
 {%- else %}
 {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
 {%- endif %}
-{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
 output {
   if [module] =~ "syslog" {
     elasticsearch {
@@ -13,10 +12,8 @@ output {
       template_name => "so-syslog"
       template => "/templates/so-syslog-template.json"
       template_overwrite => true
-      {%- if grains['role'] in ['so-node','so-heavynode'] %}
       ssl => true
       ssl_certificate_verification => false
-      {%- endif %}
     }
   }
 }

salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja

@@ -3,9 +3,8 @@
 {%- else %}
 {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
 {%- endif %}
-{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
 output {
-  if [module] =~ "osquery" {
+  if [module] =~ "osquery" and "live_query" not in [dataset] {
     elasticsearch {
       pipeline => "%{module}.%{dataset}" 
       hosts => "{{ ES }}"
@@ -13,10 +12,8 @@ output {
       template_name => "so-osquery"
       template => "/templates/so-osquery-template.json"
       template_overwrite => true
-      {%- if grains['role'] in ['so-node','so-heavynode'] %}
       ssl => true
       ssl_certificate_verification => false
-      {%- endif %}
     }
   }
 }

salt/logstash/pipelines/config/so/9101_output_osquery_livequery.conf.jinja

@@ -0,0 +1,41 @@
+{%- if grains['role'] == 'so-eval' -%}
+{%- set ES = salt['pillar.get']('manager:mainip', '') -%}
+{%- else %}
+{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
+{%- endif %}
+{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
+
+filter {
+  if [type] =~ "live_query" {
+
+    mutate {
+        rename => {
+            "[host][hostname]" => "computer_name"
+        }
+    }
+
+ prune {
+        blacklist_names => ["host"]
+      }
+    
+ split {
+   field => "rows"
+  }
+ }
+}
+
+
+output {
+  if [type] =~ "live_query" {
+    elasticsearch {
+      pipeline => "osquery.live_query" 
+      hosts => "{{ ES }}"
+      index => "so-osquery"
+      template_name => "so-osquery"
+      template => "/templates/so-osquery-template.json"
+      template_overwrite => true
+      ssl => true
+      ssl_certificate_verification => false
+    }
+  }
+}

salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja

@@ -3,7 +3,6 @@
 {%- else %}
 {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
 {%- endif %}
-{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
 output {
   if [dataset] =~ "firewall" {
     elasticsearch {
@@ -12,10 +11,8 @@ output {
       template_name => "so-firewall"
       template => "/templates/so-firewall-template.json"
       template_overwrite => true
-      {%- if grains['role'] in ['so-node','so-heavynode'] %}
       ssl => true
       ssl_certificate_verification => false
-      {%- endif %}
     }
   }
 }

salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja

@@ -3,7 +3,6 @@
 {%- else %}
 {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
 {%- endif %}
-{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
 output {
   if [module] =~ "suricata" and "import" not in [tags] {
     elasticsearch {
@@ -12,10 +11,8 @@ output {
       index => "so-ids"
       template_name => "so-ids" 
       template => "/templates/so-ids-template.json"
-      {%- if grains['role'] in ['so-node','so-heavynode'] %}
       ssl => true
       ssl_certificate_verification => false
-      {%- endif %}
     }
   }
 }

salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja

@@ -3,7 +3,6 @@
 {%- else %}
 {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
 {%- endif %}
-{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
 output {
   if "beat-ext" in [tags] and "import" not in [tags] {
     elasticsearch {
@@ -13,10 +12,8 @@ output {
       template_name => "so-beats"
       template => "/templates/so-beats-template.json"
       template_overwrite => true
-      {%- if grains['role'] in ['so-node','so-heavynode'] %}
       ssl => true
       ssl_certificate_verification => false
-      {%- endif %}
     }
   }
 }

salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja

@@ -3,7 +3,6 @@
 {%- else %}
 {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
 {%- endif %}
-{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
 output {
   if [module] =~ "ossec" {
     elasticsearch {
@@ -13,10 +12,8 @@ output {
       template_name => "so-ossec"
       template => "/templates/so-ossec-template.json"
       template_overwrite => true
-      {%- if grains['role'] in ['so-node','so-heavynode'] %}
       ssl => true
       ssl_certificate_verification => false
-      {%- endif %}
     }
   }
 }

salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja

@@ -3,7 +3,6 @@
 {%- else %}
 {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
 {%- endif %}
-{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
 output {
   if [module] =~ "strelka" {
     elasticsearch {
@@ -13,10 +12,8 @@ output {
       template_name => "so-strelka"
       template => "/templates/so-strelka-template.json"
       template_overwrite => true
-      {%- if grains['role'] in ['so-node','so-heavynode'] %}
       ssl => true
       ssl_certificate_verification => false
-      {%- endif %}
     }
   }
 }

salt/manager/files/acng/acng.conf

@@ -20,6 +20,7 @@ Remap-npm:    registry.npmjs.org
 Remap-node:   nodejs.org
 Remap-apache: file:apache_mirrors ; file:backends_apache.us
 Remap-salt:   repo.saltstack.com; https://repo.saltstack.com
+Remap-securityonion: http://repocache.securityonion.net ; file:securityonion
 # Remap-secdeb: security.debian.org
 ReportPage: acng-report.html
 # SocketPath:/var/run/apt-cacher-ng/socket
@@ -79,7 +80,7 @@ RedirMax: 6
 VfileUseRangeOps: 0
 # PassThroughPattern: private-ppa\.launchpad\.net:443$
 # PassThroughPattern: .* # this would allow CONNECT to everything
-PassThroughPattern: (download\.docker\.com:443|mirrors\.fedoraproject\.org:443|packages\.wazuh\.com:443|repo\.saltstack\.com:443|yum\.dockerproject\.org:443|download\.docker\.com:443|registry\.npmjs\.org:443|registry\.yarnpkg\.com:443)$ # yarn/npm pkg, cant to http :/
+PassThroughPattern: (repo\.securityonion\.net:443|download\.docker\.com:443|mirrors\.fedoraproject\.org:443|packages\.wazuh\.com:443|repo\.saltstack\.com:443|yum\.dockerproject\.org:443|download\.docker\.com:443|registry\.npmjs\.org:443|registry\.yarnpkg\.com:443)$ # yarn/npm pkg, cant to http :/
 # ResponseFreezeDetectTime: 500
 # ReuseConnections: 1
 # PipelineDepth: 255
@@ -89,3 +90,7 @@ PassThroughPattern: (download\.docker\.com:443|mirrors\.fedoraproject\.org:443|p
 # MaxDlSpeed: 500
 # MaxInresponsiveDlSize: 64000
 # BadRedirDetectMime: text/html
+{% set proxy = salt['pillar.get']('manager:proxy') -%}
+{% if proxy -%}
+Proxy: {{ proxy }}
+{% endif -%}

salt/manager/init.sls

@@ -18,7 +18,6 @@
 {% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %}
 {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %}
 {% set MANAGER = salt['grains.get']('master') %}
-{% set managerproxy = salt['pillar.get']('global:managerupdate', '0') %}
 {% set STRELKA_RULES = salt['pillar.get']('strelka:rules', '1') %}
 
 socore_own_saltstack:
@@ -35,8 +34,6 @@ socore_own_saltstack:
     - mode: 750
     - replace: False
 
-{% if managerproxy == 1 %}
-
 # Create the directories for apt-cacher-ng
 aptcacherconfdir:
   file.directory:
@@ -60,11 +57,12 @@ aptcacherlogdir:
     - makedirs: true
 
 # Copy the config
-
 acngcopyconf:
   file.managed:
     - name: /opt/so/conf/aptcacher-ng/etc/acng.conf
     - source: salt://manager/files/acng/acng.conf
+    - template: jinja
+    - show_changes: False
 
 # Install the apt-cacher-ng container
 so-aptcacherng:
@@ -84,8 +82,6 @@ append_so-aptcacherng_so-status.conf:
     - name: /opt/so/conf/so-status/so-status.conf
     - text: so-aptcacherng
 
-{% endif %}
-
 strelka_yara_update_old_1:
   cron.absent:
     - user: root

salt/nginx/etc/nginx.conf

@@ -25,8 +25,8 @@ events {
 
 http {
 	log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
-					  '$status $body_bytes_sent "$http_referer" '
-					  '"$http_user_agent" "$http_x_forwarded_for"';
+						'$status $body_bytes_sent "$http_referer" '
+						'"$http_user_agent" "$http_x_forwarded_for"';
 
 	access_log  /var/log/nginx/access.log  main;
 
@@ -157,7 +157,7 @@ http {
 		ssl_prefer_server_ciphers on;
 		ssl_protocols TLSv1.2;
 
-		location ~* (^/login/|^/js/.*|^/css/.*|^/images/.*) {
+		location ~* (^/login/.*|^/js/.*|^/css/.*|^/images/.*) {
 			proxy_pass            http://{{ manager_ip }}:9822;
 			proxy_read_timeout    90;
 			proxy_connect_timeout 90;
@@ -172,6 +172,8 @@ http {
 
 		location / {
 			auth_request          /auth/sessions/whoami;
+			auth_request_set      $userid $upstream_http_x_kratos_authenticated_identity_id;
+			proxy_set_header      x-user-id $userid;
 			proxy_pass            http://{{ manager_ip }}:9822/;
 			proxy_read_timeout    90;
 			proxy_connect_timeout 90;
@@ -231,15 +233,15 @@ http {
 		} 
 
 		{%- if airgap is sameas true %}
-        location /repo/ {
-            allow all;
-            sendfile on;
-            sendfile_max_chunk 1m;
-            autoindex on;
-            autoindex_exact_size off;
-            autoindex_format html;
-            autoindex_localtime on;
-        }
+		location /repo/ {
+			allow all;
+			sendfile on;
+			sendfile_max_chunk 1m;
+			autoindex on;
+			autoindex_exact_size off;
+			autoindex_format html;
+			autoindex_localtime on;
+		}
 		{%- endif %}
 		
 		location /grafana/ {