Merge pull request #4254 from Security-Onion-Solutions/2.3.51

2.3.51

README.md

@@ -1,6 +1,6 @@
-## Security Onion 2.3.40
+## Security Onion 2.3.51
 
-Security Onion 2.3.40 is here!
+Security Onion 2.3.51 is here!
 
 ## Screenshots
 

VERIFY_ISO.md

@@ -1,16 +1,17 @@
-### 2.3.40 ISO image built on 2021/03/22
+### 2.3.51 ISO image built on 2021/04/27
+
 
 ### Download and Verify
 
-2.3.40 ISO image:  
-https://download.securityonion.net/file/securityonion/securityonion-2.3.40.iso
+2.3.51 ISO image:  
+https://download.securityonion.net/file/securityonion/securityonion-2.3.51.iso
 
-MD5: FB72C0675F262A714B287BB33CE82504  
-SHA1: E8F5A9AA23990DF794611F9A178D88414F5DA81C  
-SHA256: DB125D6E770F75C3FD35ABE3F8A8B21454B7A7618C2B446D11B6AC8574601070 
+MD5: 7CFB525BEFC0A9F2ED148F5831E387FA  
+SHA1: 8CC34FCCC36822B309B8168AA706B3D1EC7F3BFD  
+SHA256: 9892C2546C9AE5A48015160F379B070F0BE30C89693B97F3F1E1592DDCE1DEE0 
 
 Signature for ISO image:  
-https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.40.iso.sig
+https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.51.iso.sig
 
 Signing key:  
 https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS  
@@ -24,22 +25,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/ma
 
 Download the signature file for the ISO:  
 ```
-wget https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.40.iso.sig
+wget https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.51.iso.sig
 ```
 
 Download the ISO image:  
 ```
-wget https://download.securityonion.net/file/securityonion/securityonion-2.3.40.iso
+wget https://download.securityonion.net/file/securityonion/securityonion-2.3.51.iso
 ```
 
 Verify the downloaded ISO image using the signature file:  
 ```
-gpg --verify securityonion-2.3.40.iso.sig securityonion-2.3.40.iso
+gpg --verify securityonion-2.3.51.iso.sig securityonion-2.3.51.iso
 ```
 
 The output should show "Good signature" and the Primary key fingerprint should match what's shown below:
 ```
-gpg: Signature made Mon 22 Mar 2021 09:35:50 AM EDT using RSA key ID FE507013
+gpg: Signature made Thu 20 May 2021 07:49:57 AM EDT using RSA key ID FE507013
 gpg: Good signature from "Security Onion Solutions, LLC <info@securityonionsolutions.com>"
 gpg: WARNING: This key is not certified with a trusted signature!
 gpg:          There is no indication that the signature belongs to the owner.

VERSION

@@ -1 +1 @@
-2.3.40
+2.3.51

pillar/docker/config.sls

@@ -1,208 +0,0 @@
-{%- set FLEETMANAGER = salt['pillar.get']('global:fleet_manager', False) -%}
-{%- set FLEETNODE = salt['pillar.get']('global:fleet_node', False) -%}
-{% set WAZUH = salt['pillar.get']('manager:wazuh', '0') %}
-{% set THEHIVE = salt['pillar.get']('manager:thehive', '0') %}
-{% set PLAYBOOK = salt['pillar.get']('manager:playbook', '0') %}
-{% set FREQSERVER = salt['pillar.get']('manager:freq', '0') %}
-{% set DOMAINSTATS = salt['pillar.get']('manager:domainstats', '0') %}
-{% set ZEEKVER = salt['pillar.get']('global:mdengine', 'COMMUNITY') %}
-{% set GRAFANA = salt['pillar.get']('manager:grafana', '0') %}
-
-eval:
-  containers:
-    - so-nginx
-    - so-telegraf
-    {% if  GRAFANA == '1' %}
-    - so-influxdb
-    - so-grafana
-    {% endif %}
-    - so-dockerregistry
-    - so-soc
-    - so-kratos
-    - so-idstools
-    {% if FLEETMANAGER %}
-    - so-mysql
-    - so-fleet
-    - so-redis
-    {% endif %}
-    - so-elasticsearch
-    - so-logstash
-    - so-kibana
-    - so-steno
-    - so-suricata
-    - so-zeek
-    - so-curator
-    - so-elastalert
-    {% if WAZUH != '0' %}
-    - so-wazuh
-    {% endif %}
-    - so-soctopus
-    {% if THEHIVE != '0' %}
-    - so-thehive
-    - so-thehive-es
-    - so-cortex
-    {% endif %}
-    {% if PLAYBOOK != '0' %}
-    - so-playbook
-    {% endif %}
-    {% if FREQSERVER != '0' %}
-    - so-freqserver
-    {% endif %}
-    {% if DOMAINSTATS != '0' %}
-    - so-domainstats
-    {% endif %}
-heavy_node:
-  containers:
-    - so-nginx
-    - so-telegraf
-    - so-redis
-    - so-logstash
-    - so-elasticsearch
-    - so-curator
-    - so-steno
-    - so-suricata
-    - so-wazuh
-    - so-filebeat
-    {% if ZEEKVER != 'SURICATA' %}
-    - so-zeek
-    {% endif %}
-helix:
-  containers:
-    - so-nginx
-    - so-telegraf
-    - so-idstools
-    - so-steno
-    - so-zeek
-    - so-redis
-    - so-logstash
-    - so-filebeat
-hot_node:
-  containers:
-    - so-nginx
-    - so-telegraf
-    - so-logstash
-    - so-elasticsearch
-    - so-curator
-manager_search:
-  containers:
-    - so-nginx
-    - so-telegraf
-    - so-soc
-    - so-kratos
-    - so-acng
-    - so-idstools
-    - so-redis
-    - so-logstash
-    - so-elasticsearch
-    - so-curator
-    - so-kibana
-    - so-elastalert
-    - so-filebeat
-    - so-soctopus
-    {% if FLEETMANAGER %}
-    - so-mysql
-    - so-fleet
-    - so-redis
-    {% endif %}
-    {% if WAZUH != '0' %}
-    - so-wazuh
-    {% endif %}
-    - so-soctopus
-    {% if THEHIVE != '0' %}
-    - so-thehive
-    - so-thehive-es
-    - so-cortex
-    {% endif %}
-    {% if PLAYBOOK != '0' %}
-    - so-playbook
-    {% endif %}
-    {% if FREQSERVER != '0' %}
-    - so-freqserver
-    {% endif %}
-    {% if DOMAINSTATS != '0' %}
-    - so-domainstats
-    {% endif %}
-manager:
-  containers:
-    - so-dockerregistry
-    - so-nginx
-    - so-telegraf
-    {% if  GRAFANA == '1' %}
-    - so-influxdb
-    - so-grafana
-    {% endif %}
-    - so-soc
-    - so-kratos
-    - so-acng
-    - so-idstools
-    - so-redis
-    - so-elasticsearch
-    - so-logstash
-    - so-kibana
-    - so-elastalert
-    - so-filebeat
-    {% if FLEETMANAGER %}
-    - so-mysql
-    - so-fleet
-    - so-redis
-    {% endif %}
-    {% if WAZUH != '0' %}
-    - so-wazuh
-    {% endif %}
-    - so-soctopus
-    {% if THEHIVE != '0' %}
-    - so-thehive
-    - so-thehive-es
-    - so-cortex
-    {% endif %}
-    {% if PLAYBOOK != '0' %}
-    - so-playbook
-    {% endif %}
-    {% if FREQSERVER != '0' %}
-    - so-freqserver
-    {% endif %}
-    {% if DOMAINSTATS != '0' %}
-    - so-domainstats
-    {% endif %}
-parser_node:
-  containers:
-    - so-nginx
-    - so-telegraf
-    - so-logstash
-search_node:
-  containers:
-    - so-nginx
-    - so-telegraf
-    - so-logstash
-    - so-elasticsearch
-    - so-curator
-    - so-filebeat
-    {% if WAZUH != '0' %}
-    - so-wazuh
-    {% endif %}
-sensor:
-  containers:
-    - so-nginx
-    - so-telegraf
-    - so-steno
-    - so-suricata
-    {% if ZEEKVER != 'SURICATA' %}
-    - so-zeek
-    {% endif %}
-    - so-wazuh
-    - so-filebeat
-warm_node:
-  containers:
-    - so-nginx
-    - so-telegraf
-    - so-elasticsearch
-fleet:
-  containers:
-    {% if FLEETNODE %}
-    - so-mysql
-    - so-fleet
-    - so-redis
-    - so-filebeat
-    - so-nginx
-    - so-telegraf
-    {% endif %}

salt/airgap/init.sls

@@ -1,71 +0,0 @@
-{% from 'allowed_states.map.jinja' import allowed_states %}
-{% if sls in allowed_states %}
-
-{% set MANAGER = salt['grains.get']('master') %}
-airgapyum:
-  file.managed:
-    - name: /etc/yum/yum.conf
-    - source: salt://airgap/files/yum.conf
-
-airgap_repo:
-  pkgrepo.managed:
-    - humanname: Airgap Repo
-    - baseurl: https://{{ MANAGER }}/repo
-    - gpgcheck: 0
-    - sslverify: 0
-
-agbase:
-  file.absent:
-    - name: /etc/yum.repos.d/CentOS-Base.repo
-
-agcr:
-  file.absent:
-    - name: /etc/yum.repos.d/CentOS-CR.repo
-
-agdebug:
-  file.absent:
-    - name: /etc/yum.repos.d/CentOS-Debuginfo.repo
-
-agfasttrack:
-  file.absent:
-    - name: /etc/yum.repos.d/CentOS-fasttrack.repo
-
-agmedia:
-  file.absent:
-    - name: /etc/yum.repos.d/CentOS-Media.repo
-
-agsources:
-  file.absent:
-    - name: /etc/yum.repos.d/CentOS-Sources.repo
-
-agvault:
-  file.absent:
-    - name: /etc/yum.repos.d/CentOS-Vault.repo
-
-agkernel:
-  file.absent:
-    - name: /etc/yum.repos.d/CentOS-x86_64-kernel.repo
-
-agepel:
-  file.absent:
-    - name: /etc/yum.repos.d/epel.repo
-
-agtesting:
-  file.absent:
-    - name: /etc/yum.repos.d/epel-testing.repo
-
-agssrepo:
-  file.absent:
-    - name: /etc/yum.repos.d/saltstack.repo
-
-agwazrepo:
-  file.absent:
-    - name: /etc/yum.repos.d/wazuh.repo
-
-{% else %}
-
-{{sls}}_state_not_allowed:
-  test.fail_without_changes:
-    - name: {{sls}}_state_not_allowed
-
-{% endif %}

salt/ca/init.sls

@@ -43,8 +43,9 @@ pki_private_key:
     - require:
       - file: /etc/pki
     - timeout: 30
-    - retry: 5
-    - interval: 30
+    - retry:
+        attempts: 5
+        interval: 30
 
 x509_pem_entries:
   module.run:

salt/common/files/99-reserved-ports.conf

@@ -1 +1 @@
-net.ipv4.ip_local_reserved_ports=55000,57314
+net.ipv4.ip_local_reserved_ports=55000,57314,47760-47860

salt/common/files/soversion

@@ -0,0 +1,2 @@
+{%- set VERSION = salt['pillar.get']('global:soversion') -%}
+{{ VERSION }}

salt/common/files/vimrc

@@ -0,0 +1,6 @@
+" Activates filetype detection
+filetype plugin indent on
+
+" Sets .sls files to use YAML syntax highlighting
+autocmd BufNewFile,BufRead *.sls set syntax=yaml
+set number

salt/common/init.sls

@@ -49,6 +49,11 @@ sosaltstackperms:
     - gid: 939
     - dir_mode: 770
 
+so_log_perms:
+  file.directory:
+    - name: /opt/so/log
+    - dir_mode: 755
+
 # Create a state directory
 statedir:
   file.directory:
@@ -64,20 +69,12 @@ salttmp:
     - group: 939
     - makedirs: True
 
-# Install epel
-{% if grains['os'] == 'CentOS' %}
-repair_yumdb:
-  cmd.run:
-    - name: 'mv -f /var/lib/rpm/__db* /tmp && yum clean all'
-    - onlyif:
-      - 'yum check-update 2>&1 | grep "Error: rpmdb open failed"'
-
-epel:
-  pkg.installed:
-    - skip_suggestions: True
-    - pkgs:
-      - epel-release
-{% endif %}
+# VIM config
+vimconfig:
+  file.managed:
+    - name: /root/.vimrc
+    - source: salt://common/files/vimrc
+    - replace: False
 
 # Install common packages
 {% if grains['os'] != 'CentOS' %}     
@@ -105,6 +102,8 @@ commonpkgs:
       - python3-mysqldb
       - python3-packaging
       - git
+      - vim
+
 heldpackages:
   pkg.installed:
     - pkgs:
@@ -143,6 +142,7 @@ commonpkgs:
       - lvm2
       - openssl
       - git
+      - vim-enhanced
 
 heldpackages:
   pkg.installed:
@@ -235,6 +235,30 @@ commonlogrotateconf:
     - month: '*'
     - dayweek: '*'
 
+# Create the status directory
+sostatusdir:
+  file.directory:
+    - name: /opt/so/log/sostatus
+    - user: 0
+    - group: 0
+    - makedirs: True
+
+sostatus_log:
+  file.managed:
+    - name: /opt/so/log/sostatus/status.log
+    - mode: 644
+    
+# Install sostatus check cron
+'/usr/sbin/so-status -q; echo $? > /opt/so/log/sostatus/status.log 2>&1':
+  cron.present:
+    - user: root
+    - minute: '*/1'
+    - hour: '*'
+    - daymonth: '*'
+    - month: '*'
+    - dayweek: '*'
+
+
 {% if role in ['eval', 'manager', 'managersearch', 'standalone'] %}
 # Lock permissions on the backup directory
 backupdir:
@@ -254,6 +278,14 @@ backupdir:
     - daymonth: '*'
     - month: '*'
     - dayweek: '*'
+{% else %}
+soversionfile:
+  file.managed:
+    - name: /etc/soversion
+    - source: salt://common/files/soversion
+    - mode: 644
+    - template: jinja
+    
 {% endif %}
 
 # Manager daemon.json
@@ -271,9 +303,10 @@ docker:
       - file: docker_daemon
 
 # Reserve OS ports for Docker proxy in case boot settings are not already applied/present
+# 55000 = Wazuh, 57314 = Strelka, 47760-47860 = Zeek
 dockerapplyports:
     cmd.run:
-      - name: if [ ! -s /etc/sysctl.d/99-reserved-ports.conf ]; then sysctl -w net.ipv4.ip_local_reserved_ports="55000,57314"; fi
+      - name: if [ ! -s /etc/sysctl.d/99-reserved-ports.conf ]; then sysctl -w net.ipv4.ip_local_reserved_ports="55000,57314,47760-47860"; fi
 
 # Reserve OS ports for Docker proxy
 dockerreserveports:

salt/common/tools/sbin/so-airgap-hotfixapply

@@ -0,0 +1,64 @@
+#!/bin/bash
+
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+. /usr/sbin/so-common
+
+UPDATE_DIR=/tmp/sohotfixapply
+
+if [ -z "$1" ]; then
+  echo "No tarball given. Please provide the filename so I can run the hotfix"
+  echo "so-airgap-hotfixapply /path/to/sohotfix.tar" 
+  exit 1
+else
+  if [ ! -f "$1" ]; then
+    echo "Unable to find $1. Make sure your path is correct and retry."
+    exit 1
+  else
+    echo "Determining if we need to apply this hotfix"
+    rm -rf $UPDATE_DIR
+    mkdir -p $UPDATE_DIR
+    tar xvf $1 -C $UPDATE_DIR
+
+    # Compare some versions
+    NEWVERSION=$(cat $UPDATE_DIR/VERSION)
+    HOTFIXVERSION=$(cat $UPDATE_DIR/HOTFIX)
+    CURRENTHOTFIX=$(cat /etc/sohotfix)
+    INSTALLEDVERSION=$(cat /etc/soversion)
+
+    if [ "$INSTALLEDVERSION" == "$NEWVERSION" ]; then
+      echo "Checking to see if there are hotfixes needed"
+      if [ "$HOTFIXVERSION" == "$CURRENTHOTFIX" ]; then
+        echo "You are already running the latest version of Security Onion."
+        rm -rf $UPDATE_DIR
+        exit 1
+      else
+        echo "We need to apply a hotfix"
+        copy_new_files
+        echo $HOTFIXVERSION > /etc/sohotfix
+        salt-call state.highstate -l info queue=True
+        echo "The Hotfix $HOTFIXVERSION has been applied"
+        # Clean up 
+        rm -rf $UPDATE_DIR
+        exit 0
+      fi
+    else
+      echo "This hotfix is not compatible with your current version. Download the latest ISO and run soup"
+      rm -rf $UPDATE_DIR
+    fi
+
+  fi
+fi

salt/common/tools/sbin/so-airgap-hotfixdownload

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# Get the latest code
+rm -rf /tmp/sohotfix
+mkdir -p /tmp/sohotfix
+cd /tmp/sohotfix
+git clone https://github.com/Security-Onion-Solutions/securityonion
+if [ ! -d "/tmp/sohotfix/securityonion" ]; then
+  echo "I was unable to get the latest code. Check your internet and try again."
+  exit 1
+else
+  echo "Looks like we have the code lets create the tarball."
+  cd /tmp/sohotfix/securityonion
+  tar cvf /tmp/sohotfix/sohotfix.tar HOTFIX VERSION salt pillar
+  echo ""
+  echo "Copy /tmp/sohotfix/sohotfix.tar to portable media and then copy it to your airgap manager."
+  exit 0
+fi

salt/common/tools/sbin/so-common

@@ -15,6 +15,8 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+DEFAULT_SALT_DIR=/opt/so/saltstack/default
+
 # Check for prerequisites
 if [ "$(id -u)" -ne 0 ]; then
 	echo "This script must be run using sudo!"
@@ -122,6 +124,20 @@ check_elastic_license() {
 	fi  
 }
 
+copy_new_files() {
+  # Copy new files over to the salt dir
+  cd $UPDATE_DIR
+  rsync -a salt $DEFAULT_SALT_DIR/
+  rsync -a pillar $DEFAULT_SALT_DIR/
+  chown -R socore:socore $DEFAULT_SALT_DIR/
+  chmod 755 $DEFAULT_SALT_DIR/pillar/firewall/addfirewall.sh
+  cd /tmp
+}
+
+disable_fastestmirror() {
+	sed -i 's/enabled=1/enabled=0/' /etc/yum/pluginconf.d/fastestmirror.conf
+}
+
 elastic_license() {
 
 read -r -d '' message <<- EOM
@@ -162,6 +178,23 @@ get_random_value() {
 	head -c 5000 /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w $length | head -n 1
 }
 
+gpg_rpm_import() {
+	if [[ "$OS" == "centos" ]]; then
+	    if [[ "$WHATWOULDYOUSAYYAHDOHERE" == "setup" ]]; then
+	        local RPMKEYSLOC="../salt/repo/client/files/centos/keys"
+	    else
+	        local RPMKEYSLOC="$UPDATE_DIR/salt/repo/client/files/centos/keys"
+	    fi
+	
+	    RPMKEYS=('RPM-GPG-KEY-EPEL-7' 'GPG-KEY-WAZUH' 'docker.pub' 'SALTSTACK-GPG-KEY.pub' 'securityonion.pub')
+
+	    for RPMKEY in "${RPMKEYS[@]}"; do
+    	        rpm --import $RPMKEYSLOC/$RPMKEY
+	        echo "Imported $RPMKEY"
+	    done
+	fi
+}
+
 header() {
 	printf '%s\n' "" "$banner" " $*" "$banner" 
 }
@@ -419,6 +452,20 @@ valid_proxy() {
 	[[ $has_prefix == true ]] && [[ $valid_url == true ]] && return 0 || return 1
 }
 
+valid_ntp_list() {
+	local string=$1
+	local ntp_arr
+	IFS="," read -r -a ntp_arr <<< "$string"
+
+	for ntp in "${ntp_arr[@]}"; do
+		if ! valid_ip4 "$ntp" && ! valid_hostname "$ntp" && ! valid_fqdn "$ntp"; then
+			return 1
+		fi
+	done
+
+	return 0
+}
+
 valid_string() {
 	local str=$1
 	local min_length=${2:-1}
@@ -440,6 +487,7 @@ wait_for_web_response() {
 	expected=$2
 	maxAttempts=${3:-300}
 	logfile=/root/wait_for_web_response.log
+	truncate -s 0 "$logfile"
 	attempt=0
 	while [[ $attempt -lt $maxAttempts ]]; do
 		attempt=$((attempt+1))

salt/common/tools/sbin/so-docker-prune

@@ -60,13 +60,17 @@ def main(quiet):
   no_prunable = True
   for t_list in grouped_tag_lists:
     try:
-      # Keep the 2 most current images
+      # Group tags by version, in case multiple images exist with the same version string
       t_list.sort(key=lambda x: Version(get_image_version(x)), reverse=True)
-      if len(t_list) <= 2:
+      grouped_t_list = [ list(it) for _,it in groupby(t_list, lambda x: get_image_version(x)) ]
+
+      # Keep the 2 most current version groups
+      if len(grouped_t_list) <= 2:
         continue
       else:
         no_prunable = False
-        for tag in t_list[2:]:
+        for group in grouped_t_list[2:]:
+          for tag in group:
             if not quiet: print(f'Removing image {tag}')
             client.images.remove(tag)
     except InvalidVersion as e:

salt/common/tools/sbin/so-elasticsearch-indices-list

@@ -0,0 +1,21 @@
+#!/bin/bash
+#
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>
+{%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%}
+
+. /usr/sbin/so-common
+
+curl -s -k -L https://{{ NODEIP }}:9200/_cat/indices?pretty

salt/common/tools/sbin/so-elasticsearch-pipeline-view

@@ -0,0 +1,25 @@
+#!/bin/bash
+#
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>
+{%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%}
+
+. /usr/sbin/so-common
+
+if [ "$1" == "" ]; then
+        curl -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/* | jq .
+else
+        curl -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/$1 | jq .
+fi

salt/common/tools/sbin/so-elasticsearch-shards-list

@@ -0,0 +1,21 @@
+#!/bin/bash
+#
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>
+{%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%}
+
+. /usr/sbin/so-common
+
+curl -s -k -L https://{{ NODEIP }}:9200/_cat/shards?pretty

salt/common/tools/sbin/so-elasticsearch-template-remove

@@ -0,0 +1,21 @@
+#!/bin/bash
+#
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>
+{%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%}
+
+. /usr/sbin/so-common
+
+curl -s -k -L -XDELETE https://{{ NODEIP }}:9200/_template/$1

salt/common/tools/sbin/so-elasticsearch-template-view

@@ -0,0 +1,25 @@
+#!/bin/bash
+#
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>
+{%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%}
+
+. /usr/sbin/so-common
+
+if [ "$1" == "" ]; then
+        curl -s -k -L https://{{ NODEIP }}:9200/_template/* | jq .
+else
+        curl -s -k -L https://{{ NODEIP }}:9200/_template/$1 | jq .
+fi

salt/common/tools/sbin/so-image-common

@@ -47,20 +47,17 @@ container_list() {
     TRUSTED_CONTAINERS=(
       "so-acng"
       "so-curator"
-      "so-domainstats"
       "so-elastalert"
       "so-elasticsearch"
       "so-filebeat"
       "so-fleet"
       "so-fleet-launcher"
-      "so-freqserver"
       "so-grafana"
       "so-idstools"
       "so-influxdb"
       "so-kibana"
       "so-kratos"
       "so-logstash"
-      "so-minio"
       "so-mysql"
       "so-nginx"
       "so-pcaptools"

salt/common/tools/sbin/so-index-list

@@ -15,4 +15,4 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-curl -X GET -k -L https://localhost:9200/_cat/indices?v
+curl -X GET -k -L "https://localhost:9200/_cat/indices?v&s=index"

salt/common/tools/sbin/so-logstash-events

@@ -0,0 +1,25 @@
+#!/bin/bash
+#
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>
+{%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%}
+
+. /usr/sbin/so-common
+
+if [ "$1" == "" ]; then
+        for i in $(curl -s -L http://{{ NODEIP }}:9600/_node/stats | jq .pipelines | jq '. | to_entries | .[].key' | sed 's/\"//g'); do echo ${i^}:; curl -s localhost:9600/_node/stats | jq .pipelines.$i.events; done
+else
+        curl -s -L http://{{ NODEIP }}:9600/_node/stats | jq .pipelines.$1.events
+fi

salt/common/tools/sbin/so-logstash-pipeline-stats

@@ -0,0 +1,25 @@
+#!/bin/bash
+#
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>
+{%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%}
+
+. /usr/sbin/so-common
+
+if [ "$1" == "" ]; then
+        curl -s -L http://{{ NODEIP }}:9600/_node/stats | jq .pipelines 
+else
+        curl -s -L http://{{ NODEIP }}:9600/_node/stats | jq .pipelines.$1
+fi

salt/common/tools/sbin/so-playbook-sync

@@ -17,4 +17,8 @@
 
 . /usr/sbin/so-common
 
+# Check to see if we are already running
+IS_RUNNING=$(ps aux | pgrep -f "so-playbook-sync" | wc -l)
+[ "$IS_RUNNING" -gt 3 ] && echo "$(date) - Multiple Playbook Sync processes already running...exiting." && exit 0
+
 docker exec so-soctopus python3 playbook_play-sync.py

salt/common/tools/sbin/so-raid-status

@@ -66,11 +66,13 @@ mkdir -p /opt/so/log/raid
   {%- if grains['sosmodel'] in ['SOSMN', 'SOSSNNV'] %}
 #check_boss_raid
 check_software_raid
-echo "osraid=$BOSSRAID nsmraid=$SWRAID" > /opt/so/log/raid/status.log
+#echo "osraid=$BOSSRAID nsmraid=$SWRAID" > /opt/so/log/raid/status.log
+echo "osraid=1 nsmraid=$SWRAID" > /opt/so/log/raid/status.log
   {%- elif grains['sosmodel'] in ['SOS1000F', 'SOS1000', 'SOSSN7200', 'SOS10K', 'SOS4000'] %}
 #check_boss_raid
 check_lsi_raid
-echo "osraid=$BOSSRAID nsmraid=$LSIRAID" > /opt/so/log/raid/status.log
+#echo "osraid=$BOSSRAID nsmraid=$LSIRAID" > /opt/so/log/raid/status.log
+echo "osraid=1 nsmraid=$LSIRAID" > /opt/so/log/raid/status.log
   {%- else %}
 exit 0
   {%- endif %}

salt/common/tools/sbin/so-sensor-clean

@@ -115,8 +115,8 @@ clean() {
 }
 
 # Check to see if we are already running
-IS_RUNNING=$(ps aux | grep "so-sensor-clean" | grep -v grep | wc -l)
-[ "$IS_RUNNING" -gt 2 ] && echo "$(date) - $IS_RUNNING sensor clean script processes running...exiting." >>$LOG && exit 0
+IS_RUNNING=$(ps aux | pgrep -f "so-sensor-clean" | wc -l)
+[ "$IS_RUNNING" -gt 3 ] && echo "$(date) - $IS_RUNNING sensor clean script processes running...exiting." >>$LOG && exit 0
 
 if [ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ]; then
 	while [ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ]; do

salt/common/tools/sbin/so-ssh-harden

@@ -4,90 +4,184 @@
 
 if [[ $1 =~ ^(-q|--quiet) ]]; then
 	quiet=true
+elif [[ $1 =~ ^(-v|--verbose) ]]; then
+	verbose=true
 fi
 
+sshd_config=/etc/ssh/sshd_config
+temp_config=/tmp/sshd_config
+
 before=
 after=
 reload_required=false
+change_header_printed=false
 
-print_sshd_t() {
+check_sshd_t() {
 	local string=$1
-	local state=$2
-	echo "${state}:"
 
 	local grep_out
 	grep_out=$(sshd -T | grep "^${string}")
 
-	if [[ $state == "Before" ]]; then
 	before=$grep_out
-	else
-		after=$grep_out
-	fi
-
-	echo $grep_out
 }
 
-print_msg() {
-	local msg=$1
-	if ! [[ $quiet ]]; then
-	printf "%s\n" \
-		"----" \
-		"$msg" \
-		"----" \
-		""
+print_diff() {
+	local diff
+	diff=$(diff -dbB <(echo $before) <(echo $after) | awk 'NR>1')
+
+	if [[ -n $diff ]]; then
+		if [[ $change_header_printed == false ]]; then
+			printf '%s\n' '' "Changes" '-------' ''
+			change_header_printed=true
+		fi
+		echo -e "$diff\n"
 	fi
 }
 
-if ! [[ $quiet ]]; then print_sshd_t "ciphers" "Before"; fi
-sshd -T | grep "^ciphers" | sed -e "s/\(3des-cbc\|aes128-cbc\|aes192-cbc\|aes256-cbc\|arcfour\|arcfour128\|arcfour256\|blowfish-cbc\|cast128-cbc\|rijndael-cbc@lysator.liu.se\)\,\?//g" >> /etc/ssh/sshd_config
-if ! [[ $quiet ]]; then
-	print_sshd_t "ciphers" "After"
-	echo ""
-fi
-
-if [[ $before != $after ]]; then
+replace_or_add() {
+	local type=$1
+	local string=$2
+	if grep -q "$type" $temp_config; then
+		sed -i "/$type .*/d" $temp_config
+	fi
+	printf "%s\n\n" "$string" >> $temp_config
 	reload_required=true
-fi
+}
 
-if ! [[ $quiet ]]; then print_sshd_t "kexalgorithms" "Before"; fi
-sshd -T | grep "^kexalgorithms" | sed -e "s/\(diffie-hellman-group14-sha1\|ecdh-sha2-nistp256\|diffie-hellman-group-exchange-sha256\|diffie-hellman-group1-sha1\|diffie-hellman-group-exchange-sha1\|ecdh-sha2-nistp521\|ecdh-sha2-nistp384\)\,\?//g" >> /etc/ssh/sshd_config
-if ! [[ $quiet ]]; then
-	print_sshd_t "kexalgorithms" "After"
-	echo ""
-fi
+test_config() {
+	local msg
+	msg=$(sshd -t -f $temp_config)
+	local ret=$?
 
-if [[ $before != $after ]]; then
-	reload_required=true
-fi
+	if [[ -n $msg ]]; then
+		echo "Error found in temp sshd config:"
+		echo $msg
+	fi
 
-if ! [[ $quiet ]]; then print_sshd_t "macs" "Before"; fi
-sshd -T | grep "^macs" | sed -e "s/\(hmac-sha2-512,\|umac-128@openssh.com,\|hmac-sha2-256,\|umac-64@openssh.com,\|hmac-sha1,\|hmac-sha1-etm@openssh.com,\|umac-64-etm@openssh.com,\|hmac-sha1\)//g" >> /etc/ssh/sshd_config
-if ! [[ $quiet ]]; then
-	print_sshd_t "macs" "After"
-	echo ""
-fi
+	return $ret
+}
 
-if [[ $before != $after ]]; then
-	reload_required=true
-fi
+main() {
+	if ! [[ $quiet ]]; then echo "Copying current config to $temp_config"; fi
+	cp $sshd_config $temp_config
 
-if ! [[ $quiet ]]; then print_sshd_t "hostkeyalgorithms" "Before"; fi
-sshd -T | grep "^hostkeyalgorithms" | sed "s|ecdsa-sha2-nistp256,||g" | sed "s|ssh-rsa,||g" >> /etc/ssh/sshd_config
-if ! [[ $quiet ]]; then
-	print_sshd_t "hostkeyalgorithms" "After"
-	echo ""
-fi
+	# Add newline to ssh for legibility
+	echo "" >> $temp_config
 
-if [[ $before != $after ]]; then
-	reload_required=true
-fi
+	# Ciphers
+	check_sshd_t "ciphers"
 
-if [[ $reload_required == true ]]; then
-	print_msg "Reloading sshd to load config changes..."
+	local bad_ciphers=(
+		"3des-cbc"
+		"aes128-cbc"
+		"aes192-cbc"
+		"aes256-cbc"
+		"arcfour"
+		"arcfour128"
+		"arcfour256"
+		"blowfish-cbc"
+		"cast128-cbc"
+	)
+
+	local cipher_string=$before
+	for cipher in "${bad_ciphers[@]}"; do
+		cipher_string=$(echo "$cipher_string" | sed "s/${cipher}\(,\|\$\)//g" | sed 's/,$//')
+	done
+
+	after=$cipher_string
+
+	if [[ $verbose ]]; then print_diff; fi
+
+	if [[ $before != "$after" ]]; then
+		replace_or_add "ciphers" "$cipher_string" && test_config || exit 1
+	fi
+
+	# KexAlgorithms
+	check_sshd_t "kexalgorithms"
+
+	local bad_kexalgs=(
+		"diffie-hellman-group-exchange-sha1"
+		"diffie-hellman-group-exchange-sha256"
+		"diffie-hellman-group1-sha1"
+		"diffie-hellman-group14-sha1"
+		"ecdh-sha2-nistp256"
+		"ecdh-sha2-nistp521"
+		"ecdh-sha2-nistp384"
+	)
+
+	local kexalg_string=$before
+	for kexalg in "${bad_kexalgs[@]}"; do
+		kexalg_string=$(echo "$kexalg_string" | sed "s/${kexalg}\(,\|\$\)//g" | sed 's/,$//')
+	done
+
+	after=$kexalg_string
+
+	if [[ $verbose ]]; then print_diff; fi
+
+	if [[ $before != "$after" ]]; then
+		replace_or_add "kexalgorithms" "$kexalg_string" && test_config || exit 1
+	fi
+
+	# Macs
+	check_sshd_t "macs"
+	
+	local bad_macs=(
+		"hmac-sha2-512"
+		"umac-128@openssh.com"
+		"hmac-sha2-256"
+		"umac-64@openssh.com"
+		"hmac-sha1"
+		"hmac-sha1-etm@openssh.com"
+		"umac-64-etm@openssh.com"
+	)
+
+	local macs_string=$before
+	for mac in "${bad_macs[@]}"; do
+		macs_string=$(echo "$macs_string" | sed "s/${mac}\(,\|\$\)//g" | sed 's/,$//')
+	done
+
+	after=$macs_string
+
+	if [[ $verbose ]]; then print_diff; fi
+
+	if [[ $before != "$after" ]]; then
+		replace_or_add "macs" "$macs_string" && test_config || exit 1
+	fi
+
+	# HostKeyAlgorithms
+	check_sshd_t "hostkeyalgorithms"
+	
+	local optional_suffix_regex_hka="\(-cert-v01@openssh.com\)\?"
+	local bad_hostkeyalg_list=(
+		"ecdsa-sha2-nistp256"
+		"ecdsa-sha2-nistp384"
+		"ecdsa-sha2-nistp521"
+		"ssh-rsa"
+		"ssh-dss"
+	)
+
+	local hostkeyalg_string=$before
+	for alg in "${bad_hostkeyalg_list[@]}"; do
+		hostkeyalg_string=$(echo "$hostkeyalg_string" | sed "s/${alg}${optional_suffix_regex_hka}\(,\|\$\)//g" | sed 's/,$//')
+	done
+
+	after=$hostkeyalg_string
+	
+	if [[ $verbose ]]; then print_diff; fi
+	
+	if [[ $before != "$after" ]]; then
+		replace_or_add "hostkeyalgorithms" "$hostkeyalg_string" && test_config || exit 1
+	fi
+
+	if [[ $reload_required == true ]]; then
+		mv -f $temp_config $sshd_config
+		if ! [[ $quiet ]]; then echo "Reloading sshd to load config changes"; fi
 		systemctl reload sshd
-fi
-
-{% if grains['os'] != 'CentOS' %}
-print_msg "[ WARNING ] Any new ssh sessions will need to remove and reaccept the ECDSA key for this server before reconnecting."
-{% endif %}
+		echo "[ WARNING ] Any new ssh sessions will need to remove and reaccept the host key fingerprint for this server before reconnecting."
+	else
+		if ! [[ $quiet ]]; then echo "No changes made to temp file, cleaning up"; fi
+		rm -f $temp_config
+	fi
+}
 
+main

salt/common/tools/sbin/soup

@@ -21,9 +21,9 @@ UPDATE_DIR=/tmp/sogh/securityonion
 INSTALLEDVERSION=$(cat /etc/soversion)
 POSTVERSION=$INSTALLEDVERSION
 INSTALLEDSALTVERSION=$(salt --versions-report | grep Salt: | awk {'print $2'})
-DEFAULT_SALT_DIR=/opt/so/saltstack/default
 BATCHSIZE=5
 SOUP_LOG=/root/soup.log
+WHATWOULDYOUSAYYAHDOHERE=soup
 
 add_common() {
   cp $UPDATE_DIR/salt/common/tools/sbin/so-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/
@@ -160,6 +160,34 @@ check_log_size_limit() {
   fi
 }
 
+check_os_updates() {
+  # Check to see if there are OS updates
+  NEEDUPDATES="We have detected missing operating system (OS) updates. Do you want to install these OS updates now? This could take a while depending on the size of your grid and how many packages are missing, but it is recommended to keep your system updated."
+  if [[ $OS == 'ubuntu' ]]; then
+    OSUPDATES=$(apt list --upgradeable | grep -v "^Listing..." | grep -v "^docker-ce" | grep -v "^wazuh-" | grep -v "^salt-" | wc -l)
+  else
+    OSUPDATES=$(yum -q list updates | wc -l)
+  fi
+  if [[ "$OSUPDATES" -gt 0 ]]; then
+      echo $NEEDUPDATES
+      echo ""
+      read -p "Press U to update OS packages (recommended), C to continue without updates, or E to exit: " confirm
+
+      if [[ "$confirm" == [cC] ]]; then
+          echo "Continuing without updating packages"
+      elif [[ "$confirm" == [uU] ]]; then
+          echo "Applying Grid Updates"
+          salt \* -b 5 state.apply patch.os queue=True
+      else
+          echo "Exiting soup"
+          exit 0
+      fi
+  else
+      echo "Looks like you have an updated OS"
+  fi
+    
+}
+
 clean_dockers() {
   # Place Holder for cleaning up old docker images
   echo "Trying to clean up old dockers."
@@ -185,16 +213,6 @@ clone_to_tmp() {
   fi
 }
 
-copy_new_files() {
-  # Copy new files over to the salt dir
-  cd $UPDATE_DIR
-  rsync -a salt $DEFAULT_SALT_DIR/
-  rsync -a pillar $DEFAULT_SALT_DIR/
-  chown -R socore:socore $DEFAULT_SALT_DIR/
-  chmod 755 $DEFAULT_SALT_DIR/pillar/firewall/addfirewall.sh
-  cd /tmp
-}
-
 generate_and_clean_tarballs() {
   local new_version
   new_version=$(cat $UPDATE_DIR/VERSION)
@@ -229,6 +247,13 @@ masterunlock() {
   fi
 }
 
+preupgrade_changes_2.3.50_repo() {
+    # We made repo changes in 2.3.50 and this prepares for that on upgrade
+    echo "Checking to see if 2.3.50 repo changes are needed."
+
+    [[ "$INSTALLEDVERSION" == 2.3.30 || "$INSTALLEDVERSION" == 2.3.40 ]] && up_2.3.3X_to_2.3.50_repo
+}
+
 preupgrade_changes() {
     # This function is to add any new pillar items if needed.
     echo "Checking to see if changes are needed."
@@ -238,6 +263,7 @@ preupgrade_changes() {
     [[ "$INSTALLEDVERSION" =~ rc.3 ]] && rc3_to_2.3.0
     [[ "$INSTALLEDVERSION" == 2.3.0 || "$INSTALLEDVERSION" == 2.3.1 || "$INSTALLEDVERSION" == 2.3.2 || "$INSTALLEDVERSION" == 2.3.10 ]] && up_2.3.0_to_2.3.20
     [[ "$INSTALLEDVERSION" == 2.3.20 || "$INSTALLEDVERSION" == 2.3.21 ]] && up_2.3.2X_to_2.3.30
+    [[ "$INSTALLEDVERSION" == 2.3.30 || "$INSTALLEDVERSION" == 2.3.40 ]] && up_2.3.3X_to_2.3.50
 }
 
 postupgrade_changes() {
@@ -408,6 +434,64 @@ up_2.3.2X_to_2.3.30() {
       sed -i "/^strelka:/a \\  repos: \n    - https://github.com/Neo23x0/signature-base" /opt/so/saltstack/local/pillar/global.sls;
   fi
   check_log_size_limit
+  INSTALLEDVERSION=2.3.30
+}
+
+up_2.3.3X_to_2.3.50_repo() {
+  echo "Performing 2.3.50 repo actions."
+  if [[ "$OS" == "centos" ]]; then
+    # Import GPG Keys
+    gpg_rpm_import
+    echo "Disabling fastestmirror."
+    disable_fastestmirror
+    echo "Deleting unneeded repo files."
+    DELREPOS=('CentOS-Base' 'CentOS-CR' 'CentOS-Debuginfo' 'docker-ce' 'CentOS-fasttrack' 'CentOS-Media' 'CentOS-Sources' 'CentOS-Vault' 'CentOS-x86_64-kernel' 'epel' 'epel-testing' 'saltstack' 'wazuh')
+
+    for DELREPO in "${DELREPOS[@]}"; do
+      if [[ -f "/etc/yum.repos.d/$DELREPO.repo" ]]; then
+        echo "Deleting $DELREPO.repo"
+        rm -f "/etc/yum.repos.d/$DELREPO.repo"
+      fi
+    done
+    if [ $is_airgap -eq 1 ]; then
+      # Copy the new repo file if not airgap
+      cp $UPDATE_DIR/salt/repo/client/files/centos/securityonion.repo /etc/yum.repos.d/
+      yum clean all
+      yum repolist
+    fi
+  fi
+}
+
+up_2.3.3X_to_2.3.50() {
+  
+  cat <<EOF > /tmp/supersed.txt
+/so-zeek:/ {
+  p;
+  n;
+  /shards:/ {
+    p;
+    n;
+    /warm:/ {
+      p;
+      n;
+      /close:/ {
+        s/close: 365/close: 45/;
+        p;
+        n;
+        /delete:/ {
+          s/delete: 45/delete: 365/;
+          p;
+          d;
+        }
+      }
+    }
+  }
+}
+p;
+EOF
+  sed -n -i -f /tmp/supersed.txt /opt/so/saltstack/local/pillar/global.sls
+  rm /tmp/supersed.txt
+  INSTALLEDVERSION=2.3.50
 }
 
 verify_upgradespace() {
@@ -477,16 +561,28 @@ update_version() {
   # Update the version to the latest
   echo "Updating the Security Onion version file."
   echo $NEWVERSION > /etc/soversion
+  echo $HOTFIXVERSION > /etc/sohotfix
   sed -i "/  soversion:/c\  soversion: $NEWVERSION" /opt/so/saltstack/local/pillar/global.sls
 }
 
 upgrade_check() {
   # Let's make sure we actually need to update.
   NEWVERSION=$(cat $UPDATE_DIR/VERSION)
+  HOTFIXVERSION=$(cat $UPDATE_DIR/HOTFIX)
+  CURRENTHOTFIX=$(cat /etc/sohotfix 2>/dev/null)
   if [ "$INSTALLEDVERSION" == "$NEWVERSION" ]; then
+    echo "Checking to see if there are hotfixes needed"
+    if [ "$HOTFIXVERSION" == "$CURRENTHOTFIX" ]; then
       echo "You are already running the latest version of Security Onion."
       exit 0
+    else
+      echo "We need to apply a hotfix"
+      is_hotfix=true
     fi
+  else
+    is_hotfix=false
+  fi 
+
 }
 
 upgrade_check_salt() {
@@ -502,22 +598,18 @@ upgrade_salt() {
   echo "Performing upgrade of Salt from $INSTALLEDSALTVERSION to $NEWSALTVERSION."
   echo ""
   # If CentOS
-  if [ "$OS" == "centos" ]; then
+  if [[ $OS == 'centos' ]]; then
     echo "Removing yum versionlock for Salt."
     echo ""
     yum versionlock delete "salt-*"
     echo "Updating Salt packages and restarting services."
     echo ""
-    if [ $is_airgap -eq 0 ]; then
     sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -r -F -M -x python3 stable "$NEWSALTVERSION"
-    else 
-      sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -F -M -x python3 stable "$NEWSALTVERSION"
-    fi
     echo "Applying yum versionlock for Salt."
     echo ""
     yum versionlock add "salt-*"
   # Else do Ubuntu things
-  elif [ "$OS" == "ubuntu" ]; then
+  elif [[ $OS == 'ubuntu' ]]; then
     echo "Removing apt hold for Salt."
     echo ""
     apt-mark unhold "salt-common"
@@ -597,7 +689,7 @@ else
   rm -rf $UPDATE_DIR
   clone_to_tmp
 fi
-
+check_os_updates
 echo ""
 echo "Verifying we have the latest soup script."
 verify_latest_update_script
@@ -618,122 +710,134 @@ upgrade_space
 echo "Checking for Salt Master and Minion updates."
 upgrade_check_salt
 
-echo ""
-echo "Performing upgrade from Security Onion $INSTALLEDVERSION to Security Onion $NEWVERSION."
-echo ""
-echo "Updating dockers to $NEWVERSION."
-if [ $is_airgap -eq 0 ]; then
-  airgap_update_dockers
-else
-  update_registry
-  update_docker_containers "soup"
-fi
-echo ""
-echo "Stopping Salt Minion service."
-systemctl stop salt-minion
-echo "Killing any remaining Salt Minion processes."
-pkill -9 -ef /usr/bin/salt-minion
-echo ""
-echo "Stopping Salt Master service."
-systemctl stop salt-master
-echo ""
 
-# Does salt need upgraded. If so update it.
-if [ "$UPGRADESALT" == "1" ]; then
-  echo "Upgrading Salt"
-  # Update the repo files so it can actually upgrade
+if [ "$is_hotfix" == "true" ]; then
+  echo "Applying $HOTFIXVERSION" 
+  copy_new_files
+  echo ""
+  update_version
+  salt-call state.highstate -l info queue=True
+  
+else
+  echo ""
+  echo "Performing upgrade from Security Onion $INSTALLEDVERSION to Security Onion $NEWVERSION."
+  echo ""
+
+  echo "Updating dockers to $NEWVERSION."
   if [ $is_airgap -eq 0 ]; then
+    airgap_update_dockers
     update_centos_repo
     yum clean all
+    check_os_updates
+  else
+    update_registry
+    update_docker_containers "soup"
   fi
-  upgrade_salt
-fi
 
-echo "Checking if Salt was upgraded."
-echo ""
-# Check that Salt was upgraded
-if [[ $(salt --versions-report | grep Salt: | awk {'print $2'}) != "$NEWSALTVERSION" ]]; then
+  echo ""
+  echo "Stopping Salt Minion service."
+  systemctl stop salt-minion
+  echo "Killing any remaining Salt Minion processes."
+  pkill -9 -ef /usr/bin/salt-minion
+  echo ""
+  echo "Stopping Salt Master service."
+  systemctl stop salt-master
+  echo ""
+
+  preupgrade_changes_2.3.50_repo
+
+  # Does salt need upgraded. If so update it.
+  if [ "$UPGRADESALT" == "1" ]; then
+    echo "Upgrading Salt"
+    # Update the repo files so it can actually upgrade
+    upgrade_salt
+  fi
+
+  echo "Checking if Salt was upgraded."
+  echo ""
+  # Check that Salt was upgraded
+  SALTVERSIONPOSTUPGRADE=$(salt --versions-report | grep Salt: | awk {'print $2'})
+  if [[ "$SALTVERSIONPOSTUPGRADE" != "$NEWSALTVERSION" ]]; then
     echo "Salt upgrade failed. Check of indicators of failure in $SOUP_LOG."
     echo "Once the issue is resolved, run soup again."
     echo "Exiting."
     echo ""
     exit 1
-else
+  else
     echo "Salt upgrade success."
     echo ""
-fi
+  fi
 
-preupgrade_changes
-echo ""
+  preupgrade_changes
+  echo ""
 
-if [ $is_airgap -eq 0 ]; then
+  if [ $is_airgap -eq 0 ]; then
     echo "Updating Rule Files to the Latest."
     update_airgap_rules
-fi
+  fi
 
-# Only update the repo if its airgap
-if [[ $is_airgap -eq 0 ]] && [[ "$UPGRADESALT" != "1" ]]; then
-update_centos_repo
-fi
+  # Only update the repo if its airgap
+  if [[ $is_airgap -eq 0 ]] && [[ "$UPGRADESALT" != "1" ]]; then
+  update_centos_repo
+  fi
 
-echo ""
-echo "Copying new Security Onion code from $UPDATE_DIR to $DEFAULT_SALT_DIR."
-copy_new_files
-echo ""
-update_version
+  echo ""
+  echo "Copying new Security Onion code from $UPDATE_DIR to $DEFAULT_SALT_DIR."
+  copy_new_files
+  echo ""
+  update_version
 
-echo ""
-echo "Locking down Salt Master for upgrade"
-masterlock
+  echo ""
+  echo "Locking down Salt Master for upgrade"
+  masterlock
 
-echo ""
-echo "Starting Salt Master service."
-systemctl start salt-master
+  echo ""
+  echo "Starting Salt Master service."
+  systemctl start salt-master
 
-# Only regenerate osquery packages if Fleet is enabled
-FLEET_MANAGER=$(lookup_pillar fleet_manager)
-FLEET_NODE=$(lookup_pillar fleet_node)
-if [[ "$FLEET_MANAGER" == "True" || "$FLEET_NODE" == "True" ]]; then
+  # Only regenerate osquery packages if Fleet is enabled
+  FLEET_MANAGER=$(lookup_pillar fleet_manager)
+  FLEET_NODE=$(lookup_pillar fleet_node)
+  if [[ "$FLEET_MANAGER" == "True" || "$FLEET_NODE" == "True" ]]; then
     echo ""
     echo "Regenerating Osquery Packages.... This will take several minutes."
     salt-call state.apply fleet.event_gen-packages -l info queue=True
     echo ""
-fi
-
-echo ""
-echo "Running a highstate to complete the Security Onion upgrade on this manager. This could take several minutes."
-salt-call state.highstate -l info queue=True
-echo ""
-echo "Upgrade from $INSTALLEDVERSION to $NEWVERSION complete."
-
-echo ""
-echo "Stopping Salt Master to remove ACL"
-systemctl stop salt-master
-
-masterunlock
-
-echo ""
-echo "Starting Salt Master service."
-systemctl start salt-master
-echo "Running a highstate. This could take several minutes."
-salt-call state.highstate -l info queue=True
-postupgrade_changes
-unmount_update
-thehive_maint
-
-if [ "$UPGRADESALT" == "1" ]; then
-  echo ""
-  echo "Upgrading Salt on the remaining Security Onion nodes from $INSTALLEDSALTVERSION to $NEWSALTVERSION."
-  if [ $is_airgap -eq 0 ]; then
-    salt -C 'not *_eval and not *_helixsensor and not *_manager and not *_managersearch and not *_standalone' cmd.run "yum clean all"
   fi
-  salt -C 'not *_eval and not *_helixsensor and not *_manager and not *_managersearch and not *_standalone' -b $BATCHSIZE state.apply salt.minion queue=True
+
   echo ""
-fi
+  echo "Running a highstate to complete the Security Onion upgrade on this manager. This could take several minutes."
+  salt-call state.highstate -l info queue=True
+  echo ""
+  echo "Upgrade from $INSTALLEDVERSION to $NEWVERSION complete."
 
-check_sudoers
+  echo ""
+  echo "Stopping Salt Master to remove ACL"
+  systemctl stop salt-master
 
-if [[ -n $lsl_msg ]]; then
+  masterunlock
+
+  echo ""
+  echo "Starting Salt Master service."
+  systemctl start salt-master
+  echo "Running a highstate. This could take several minutes."
+  salt-call state.highstate -l info queue=True
+  postupgrade_changes
+  unmount_update
+  thehive_maint
+
+  if [ "$UPGRADESALT" == "1" ]; then
+    if [ $is_airgap -eq 0 ]; then
+      echo ""
+      echo "Cleaning repos on remote Security Onion nodes."
+      salt -C 'not *_eval and not *_helixsensor and not *_manager and not *_managersearch and not *_standalone and G@os:CentOS' cmd.run "yum clean all"
+      echo ""
+    fi
+  fi
+
+  check_sudoers
+
+  if [[ -n $lsl_msg ]]; then
     case $lsl_msg in
       'distributed')
         echo "[INFO] The value of log_size_limit in any heavy node minion pillars may be incorrect."
@@ -747,14 +851,16 @@ if [[ -n $lsl_msg ]]; then
         echo " -> File: /opt/so/saltstack/local/pillar/minions/${lsl_details[2]}.sls"
       ;;
     esac
-fi
+  fi
 
-NUM_MINIONS=$(ls /opt/so/saltstack/local/pillar/minions/*_*.sls | wc -l)
+  NUM_MINIONS=$(ls /opt/so/saltstack/local/pillar/minions/*_*.sls | wc -l)
 
-if [ $NUM_MINIONS -gt 1 ]; then
+  if [ $NUM_MINIONS -gt 1 ]; then
 
     cat << EOF
     
+  
+  
 This appears to be a distributed deployment. Other nodes should update themselves at the next Salt highstate (typically within 15 minutes). Do not manually restart anything until you know that all the search/heavy nodes in your deployment are updated. This is especially important if you are using true clustering for Elasticsearch.
 
 Each minion is on a random 15 minute check-in period and things like network bandwidth can be a factor in how long the actual upgrade takes. If you have a heavy node on a slow link, it is going to take a while to get the containers to it. Depending on what changes happened between the versions, Elasticsearch might not be able to talk to said heavy node until the update is complete.
@@ -762,9 +868,12 @@ Each minion is on a random 15 minute check-in period and things like network ban
 If it looks like you’re missing data after the upgrade, please avoid restarting services and instead make sure at least one search node has completed its upgrade. The best way to do this is to run 'sudo salt-call state.highstate' from a search node and make sure there are no errors. Typically if it works on one node it will work on the rest. Forward nodes are less complex and will update as they check in so you can monitor those from the Grid section of SOC.
 
 For more information, please see https://docs.securityonion.net/en/2.3/soup.html#distributed-deployments.
+
 EOF
 
+  fi
 fi
+
 echo "### soup has been served at `date` ###"
 }
 
@@ -776,8 +885,6 @@ Please review the following for more information about the update process and re
 https://docs.securityonion.net/soup
 https://blog.securityonion.net
 
-Please note that soup only updates Security Onion components and does NOT update the underlying operating system (OS). When you installed Security Onion, there was an option to automatically update the OS packages. If you did not enable this option, then you will want to ensure that the OS is fully updated before running soup.
-
 Press Enter to continue or Ctrl-C to cancel.
 
 EOF

salt/curator/files/bin/so-curator-closed-delete-delete

@@ -34,7 +34,7 @@ overlimit() {
 
 closedindices() {
 
-        INDICES=$(curl -s -k {% if grains['role'] in ['so-node','so-heavynode'] %}https://{% endif %}{{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/_cat/indices?h=index\&expand_wildcards=closed 2> /dev/null)
+        INDICES=$(curl -s -k https://{{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/_cat/indices?h=index\&expand_wildcards=closed 2> /dev/null)
         [ $? -eq 1 ] && return false 
         echo ${INDICES} | grep -q -E "(logstash-|so-)"
 }
@@ -49,10 +49,10 @@ while overlimit && closedindices; do
 	# First, get the list of closed indices using _cat/indices?h=index\&expand_wildcards=closed.
 	# Then, sort by date by telling sort to use hyphen as delimiter and then sort on the third field.
 	# Finally, select the first entry in that sorted list.
-	OLDEST_INDEX=$(curl -s -k {% if grains['role'] in ['so-node','so-heavynode'] %}https://{% endif %}{{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/_cat/indices?h=index\&expand_wildcards=closed | grep -E "(logstash-|so-)" | sort -t- -k3 | head -1)
+	OLDEST_INDEX=$(curl -s -k https://{{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/_cat/indices?h=index\&expand_wildcards=closed | grep -E "(logstash-|so-)" | sort -t- -k3 | head -1)
 	
 	# Now that we've determined OLDEST_INDEX, ask Elasticsearch to delete it.
-	curl -XDELETE -k {% if grains['role'] in ['so-node','so-heavynode'] %}https://{% endif %}{{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/${OLDEST_INDEX}
+	curl -XDELETE -k https://{{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/${OLDEST_INDEX}
 
 	# Finally, write a log entry that says we deleted it.
 	echo "$(date) - Used disk space exceeds LOG_SIZE_LIMIT ({{LOG_SIZE_LIMIT}} GB) - Index ${OLDEST_INDEX} deleted ..." >> ${LOG}

salt/elastalert/files/modules/so/playbook-es.py

@@ -17,7 +17,7 @@ class PlaybookESAlerter(Alerter):
     def alert(self, matches):
        for match in matches:
             today = strftime("%Y.%m.%d", gmtime())
-            timestamp = strftime("%Y-%m-%d"'T'"%H:%M:%S", gmtime())
+            timestamp = strftime("%Y-%m-%d"'T'"%H:%M:%S"'.000Z', gmtime())
             headers = {"Content-Type": "application/json"}
             payload = {"rule": { "name": self.rule['play_title'],"case_template": self.rule['play_id'],"uuid": self.rule['play_id'],"category": self.rule['rule.category']},"event":{ "severity": self.rule['event.severity'],"module": self.rule['event.module'],"dataset": self.rule['event.dataset'],"severity_label": self.rule['sigma_level']},"kibana_pivot": self.rule['kibana_pivot'],"soc_pivot": self.rule['soc_pivot'],"play_url": self.rule['play_url'],"sigma_level": self.rule['sigma_level'],"event_data": match, "@timestamp": timestamp}
             url = f"https://{self.rule['elasticsearch_host']}/so-playbook-alerts-{today}/_doc/"

salt/elasticsearch/files/ingest/osquery.query_result

@@ -9,6 +9,7 @@
     { "rename":      { "if": "!(ctx.error?.eventdata_parsing == true)", "field": "unparsed.EventData", "target_field": "winlog.event_data",       "ignore_missing": true, "ignore_failure": true  } },   
     { "rename":      { "field": "winlog.source",               "target_field": "winlog.channel",       "ignore_missing": true  } },    
     { "rename":      { "field": "winlog.eventid",               "target_field": "winlog.event_id",       "ignore_missing": true  } },   
+    { "rename":      { "field": "winlog.datetime",      "target_field": "winlog.systemTime",     "ignore_missing": true } },
     { "pipeline":    { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational'",   "name": "sysmon"  }  },
     { "pipeline":    { "if": "ctx.winlog?.channel != 'Microsoft-Windows-Sysmon/Operational' && ctx.containsKey('winlog')",  "name":"win.eventlogs" }  },
     { "set":           { "field": "event.module", "value": "osquery", "override": false }  },

salt/elasticsearch/files/ingest/win.eventlogs

@@ -4,8 +4,8 @@
     { "set":           { "if": "ctx.winlog?.channel != null",   "field": "event.module", "value": "windows_eventlog", "override": false, "ignore_failure": true }  },
     { "set":           { "if": "ctx.winlog?.channel != null",   "field": "event.dataset", "value": "{{winlog.channel}}", "override": true }  },
     { "set":           { "if": "ctx.winlog?.computer_name != null",   "field": "observer.name", "value": "{{winlog.computer_name}}", "override": true }  },
-    { "rename":         { "if": "ctx.winlog?.systemTime != null", "field": "@timestamp",      "target_field": "ingest.timestamp",     "ignore_missing": true } },
-    { "set":           { "if": "ctx.winlog?.systemTime != null",   "field": "@timestamp", "value": "{{winlog.systemTime}}", "override": true }  },
+    { "rename":       { "if": "ctx.winlog?.systemTime != null", "field": "@timestamp",      "target_field": "event.ingested",     "ignore_missing": true } },
+    { "date":         { "if": "ctx.winlog?.systemTime != null", "field": "winlog.systemTime",      "formats": ["yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSS'Z'","yyyy-MM-dd'T'HH:mm:ss.SSSSSSS'Z'"] } },    
     { "set":           { "field": "event.code", "value": "{{winlog.event_id}}", "override": true }  },  
     { "set":           { "field": "event.category", "value": "host", "override": true }  },   
     { "rename": 	   { "field": "winlog.event_data.SubjectUserName", 			  "target_field": "user.name",		"ignore_failure": true,   "ignore_missing": true 	} },

salt/elasticsearch/templates/so/so-common-template.json

@@ -267,7 +267,12 @@
         },
         "ingest":{
           "type":"object",
-          "dynamic": true
+          "dynamic": true,
+          "properties":{
+            "timestamp":{
+              "type":"date"
+            }
+          }
         },
 	"intel":{
           "type":"object",

salt/filebeat/etc/filebeat.yml

@@ -493,12 +493,13 @@ setup.template.enabled: false
 # append ?pretty to the URL.
 
 # Defines if the HTTP endpoint is enabled.
-#http.enabled: false
+http.enabled: true
 
 # The HTTP endpoint will bind to this hostname or IP address. It is recommended to use only localhost.
-#http.host: localhost
+http.host: 0.0.0.0
 
 # Port on which the HTTP endpoint will bind. Default is 5066.
+http.port: 5066
 
 queue.mem.events: {{ FBMEMEVENTS }}
 queue.mem.flush.min_events: {{ FBMEMFLUSHMINEVENTS }}

salt/filebeat/init.sls

@@ -74,6 +74,7 @@ so-filebeat:
     - port_bindings:
         - 0.0.0.0:514:514/udp
         - 0.0.0.0:514:514/tcp
+        - 0.0.0.0:5066:5066/tcp
     - watch:
       - file: /opt/so/conf/filebeat/etc/filebeat.yml
 

salt/firewall/map.jinja

@@ -18,14 +18,18 @@
 
 {# This block translate the portgroups defined in the pillar to what is defined my portgroups.yaml and portgroups.local.yaml #}
 {% if salt['pillar.get']('firewall:assigned_hostgroups:chain') %}
+  {% set translated_pillar_assigned_hostgroups = {'chain': {}} %}
 
   {% for chain, hg in salt['pillar.get']('firewall:assigned_hostgroups:chain').items() %}
     {% for pillar_hostgroup, pillar_portgroups in salt['pillar.get']('firewall:assigned_hostgroups:chain')[chain].hostgroups.items() %}
-      {% do translated_pillar_assigned_hostgroups.update({"chain": {chain: {"hostgroups": {pillar_hostgroup: {"portgroups": []}}}}}) %}
+      {% if translated_pillar_assigned_hostgroups.chain[chain] is defined %}
+        {% do translated_pillar_assigned_hostgroups.chain[chain].hostgroups.update({pillar_hostgroup: {"portgroups": []}}) %}
+      {% else %}
+        {% do translated_pillar_assigned_hostgroups.chain.update({chain: {"hostgroups": {pillar_hostgroup: {"portgroups": []}}}}) %}
+      {% endif %}
       {% for pillar_portgroup in pillar_portgroups.portgroups %}
         {% set pillar_portgroup = pillar_portgroup.split('.') | last %}
         {% do translated_pillar_assigned_hostgroups.chain[chain].hostgroups[pillar_hostgroup].portgroups.append(defined_portgroups[pillar_portgroup]) %}
-
       {% endfor %}
     {% endfor %}
   {% endfor %}
@@ -39,7 +43,6 @@
   {% set assigned_hostgroups = default_assigned_hostgroups.role[role] %}
 {% endif %}
 
-
 {% if translated_pillar_assigned_hostgroups %}
   {% do salt['defaults.merge'](assigned_hostgroups, translated_pillar_assigned_hostgroups, merge_lists=True, in_place=True) %}
 {% endif %}

salt/firewall/portgroups.yaml

@@ -18,6 +18,9 @@ firewall:
       beats_5644:
         tcp:
           - 5644
+      beats_5066:
+        tcp:
+          - 5066
       cortex:
         tcp:
           - 9001

salt/grafana/dashboards/manager/manager.json

@@ -4322,139 +4322,6 @@
         "align": false,
         "alignLevel": null
       }
-    },
-    {
-      "aliasColors": {},
-      "bars": false,
-      "dashLength": 10,
-      "dashes": false,
-      "datasource": "InfluxDB",
-      "fieldConfig": {
-        "defaults": {
-          "custom": {}
-        },
-        "overrides": []
-      },
-      "fill": 1,
-      "fillGradient": 0,
-      "gridPos": {
-        "h": 6,
-        "w": 8,
-        "x": 16,
-        "y": 31
-      },
-      "hiddenSeries": false,
-      "id": 76,
-      "legend": {
-        "avg": false,
-        "current": false,
-        "max": false,
-        "min": false,
-        "show": false,
-        "total": false,
-        "values": false
-      },
-      "lines": true,
-      "linewidth": 1,
-      "nullPointMode": "connected",
-      "options": {
-        "alertThreshold": true
-      },
-      "percentage": false,
-      "pluginVersion": "7.3.4",
-      "pointradius": 2,
-      "points": false,
-      "renderer": "flot",
-      "seriesOverrides": [],
-      "spaceLength": 10,
-      "stack": false,
-      "steppedLine": false,
-      "targets": [
-        {
-          "alias": "EPS",
-          "groupBy": [
-            {
-              "params": [
-                "$__interval"
-              ],
-              "type": "time"
-            },
-            {
-              "params": [
-                "null"
-              ],
-              "type": "fill"
-            }
-          ],
-          "measurement": "esteps",
-          "orderByTime": "ASC",
-          "policy": "default",
-          "queryType": "randomWalk",
-          "refId": "A",
-          "resultFormat": "time_series",
-          "select": [
-            [
-              {
-                "params": [
-                  "eps"
-                ],
-                "type": "field"
-              },
-              {
-                "params": [],
-                "type": "mean"
-              }
-            ]
-          ],
-          "tags": [
-            {
-              "key": "host",
-              "operator": "=",
-              "value": "{{ SERVERNAME }}"
-            }
-          ]
-        }
-      ],
-      "thresholds": [],
-      "timeFrom": null,
-      "timeRegions": [],
-      "timeShift": null,
-      "title": "{{ SERVERNAME }} - Estimated EPS",
-      "tooltip": {
-        "shared": true,
-        "sort": 0,
-        "value_type": "individual"
-      },
-      "type": "graph",
-      "xaxis": {
-        "buckets": null,
-        "mode": "time",
-        "name": null,
-        "show": true,
-        "values": []
-      },
-      "yaxes": [
-        {
-          "format": "short",
-          "label": "EPS",
-          "logBase": 1,
-          "max": null,
-          "min": null,
-          "show": true
-        },
-        {
-          "format": "short",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": null,
-          "show": false
-        }
-      ],
-      "yaxis": {
-        "align": false,
-        "alignLevel": null
-      }
     }
   ],
   "refresh": false,

salt/grafana/dashboards/managersearch/managersearch.json

@@ -5157,7 +5157,7 @@
               "type": "fill"
             }
           ],
-          "measurement": "esteps",
+          "measurement": "consumptioneps",
           "orderByTime": "ASC",
           "policy": "default",
           "queryType": "randomWalk",

salt/grafana/dashboards/standalone/standalone.json

@@ -5562,7 +5562,7 @@
               "type": "fill"
             }
           ],
-          "measurement": "esteps",
+          "measurement": "consumptioneps",
           "orderByTime": "ASC",
           "policy": "default",
           "queryType": "randomWalk",

salt/grafana/init.sls

@@ -11,7 +11,7 @@
 {% set GRAFANA_SETTINGS = salt['grains.filter_by'](default_settings, default='grafana', merge=salt['pillar.get']('grafana', {})) %}
 
 
-{% if grains['role'] in ['so-manager', 'so-managersearch', 'so-eval', 'so-standalone'] and GRAFANA == 1 %}
+{% if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone'] or (grains.role == 'so-eval' and GRAFANA == 1) %}
 
 # Grafana all the things
 grafanadir:

salt/influxdb/etc/influxdb.conf

@@ -233,7 +233,7 @@
   # enabled = true
 
   # Determines whether the Flux query endpoint is enabled.
-  # flux-enabled = false
+  flux-enabled = true
 
   # The bind address used by the HTTP service.
   # bind-address = ":8086"

salt/influxdb/init.sls

@@ -6,7 +6,7 @@
 {% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %}
 {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %}
 
-{% if grains['role'] in ['so-manager', 'so-managersearch', 'so-eval', 'so-standalone'] and GRAFANA == 1 %}
+{% if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone'] or (grains.role == 'so-eval' and GRAFANA == 1) %}
 
 # Influx DB
 influxconfdir:

salt/manager/files/acng/acng.conf

@@ -20,6 +20,7 @@ Remap-npm:    registry.npmjs.org
 Remap-node:   nodejs.org
 Remap-apache: file:apache_mirrors ; file:backends_apache.us
 Remap-salt:   repo.saltstack.com; https://repo.saltstack.com
+Remap-securityonion: http://repocache.securityonion.net ; file:securityonion
 # Remap-secdeb: security.debian.org
 ReportPage: acng-report.html
 # SocketPath:/var/run/apt-cacher-ng/socket
@@ -79,7 +80,7 @@ RedirMax: 6
 VfileUseRangeOps: 0
 # PassThroughPattern: private-ppa\.launchpad\.net:443$
 # PassThroughPattern: .* # this would allow CONNECT to everything
-PassThroughPattern: (download\.docker\.com:443|mirrors\.fedoraproject\.org:443|packages\.wazuh\.com:443|repo\.saltstack\.com:443|yum\.dockerproject\.org:443|download\.docker\.com:443|registry\.npmjs\.org:443|registry\.yarnpkg\.com:443)$ # yarn/npm pkg, cant to http :/
+PassThroughPattern: (repo\.securityonion\.net:443|download\.docker\.com:443|mirrors\.fedoraproject\.org:443|packages\.wazuh\.com:443|repo\.saltstack\.com:443|yum\.dockerproject\.org:443|download\.docker\.com:443|registry\.npmjs\.org:443|registry\.yarnpkg\.com:443)$ # yarn/npm pkg, cant to http :/
 # ResponseFreezeDetectTime: 500
 # ReuseConnections: 1
 # PipelineDepth: 255
@@ -89,3 +90,7 @@ PassThroughPattern: (download\.docker\.com:443|mirrors\.fedoraproject\.org:443|p
 # MaxDlSpeed: 500
 # MaxInresponsiveDlSize: 64000
 # BadRedirDetectMime: text/html
+{% set proxy = salt['pillar.get']('manager:proxy') -%}
+{% if proxy -%}
+Proxy: {{ proxy }}
+{% endif -%}

salt/manager/init.sls

@@ -18,7 +18,6 @@
 {% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %}
 {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %}
 {% set MANAGER = salt['grains.get']('master') %}
-{% set managerproxy = salt['pillar.get']('global:managerupdate', '0') %}
 {% set STRELKA_RULES = salt['pillar.get']('strelka:rules', '1') %}
 
 socore_own_saltstack:
@@ -35,8 +34,6 @@ socore_own_saltstack:
     - mode: 750
     - replace: False
 
-{% if managerproxy == 1 %}
-
 # Create the directories for apt-cacher-ng
 aptcacherconfdir:
   file.directory:
@@ -60,11 +57,12 @@ aptcacherlogdir:
     - makedirs: true
 
 # Copy the config
-
 acngcopyconf:
   file.managed:
     - name: /opt/so/conf/aptcacher-ng/etc/acng.conf
     - source: salt://manager/files/acng/acng.conf
+    - template: jinja
+    - show_changes: False
 
 # Install the apt-cacher-ng container
 so-aptcacherng:
@@ -84,8 +82,6 @@ append_so-aptcacherng_so-status.conf:
     - name: /opt/so/conf/so-status/so-status.conf
     - text: so-aptcacherng
 
-{% endif %}
-
 strelka_yara_update_old_1:
   cron.absent:
     - user: root

salt/nginx/etc/nginx.conf

@@ -157,7 +157,7 @@ http {
 		ssl_prefer_server_ciphers on;
 		ssl_protocols TLSv1.2;
 
-		location ~* (^/login/|^/js/.*|^/css/.*|^/images/.*) {
+		location ~* (^/login/.*|^/js/.*|^/css/.*|^/images/.*) {
 			proxy_pass            http://{{ manager_ip }}:9822;
 			proxy_read_timeout    90;
 			proxy_connect_timeout 90;

salt/repo/client/files/centos/keys/GPG-KEY-WAZUH

@@ -0,0 +1,52 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+Version: GnuPG v2.0.22 (GNU/Linux)
+
+mQINBFeeyYwBEACyf4VwV8c2++J5BmCl6ofLCtSIW3UoVrF4F+P19k/0ngnSfjWb
+8pSWB11HjZ3Mr4YQeiD7yY06UZkrCXk+KXDlUjMK3VOY7oNPkqzNaP6+8bDwj4UA
+hADMkaXBvWooGizhCoBtDb1bSbHKcAnQ3PTdiuaqF5bcyKk8hv939CHulL2xH+BP
+mmTBi+PM83pwvR+VRTOT7QSzf29lW1jD79v4rtXHJs4KCz/amT/nUm/tBpv3q0sT
+9M9rH7MTQPdqvzMl122JcZST75GzFJFl0XdSHd5PAh2mV8qYak5NYNnwA41UQVIa
++xqhSu44liSeZWUfRdhrQ/Nb01KV8lLAs11Sz787xkdF4ad25V/Rtg/s4UXt35K3
+klGOBwDnzPgHK/OK2PescI5Ve1z4x1C2bkGze+gk/3IcfGJwKZDfKzTtqkZ0MgpN
+7RGghjkH4wpFmuswFFZRyV+s7jXYpxAesElDSmPJ0O07O4lQXQMROE+a2OCcm0eF
+3+Cr6qxGtOp1oYMOVH0vOLYTpwOkAM12/qm7/fYuVPBQtVpTojjV5GDl2uGq7p0o
+h9hyWnLeNRbAha0px6rXcF9wLwU5n7mH75mq5clps3sP1q1/VtP/Fr84Lm7OGke4
+9eD+tPNCdRx78RNWzhkdQxHk/b22LCn1v6p1Q0qBco9vw6eawEkz1qwAjQARAQAB
+tDFXYXp1aC5jb20gKFdhenVoIFNpZ25pbmcgS2V5KSA8c3VwcG9ydEB3YXp1aC5j
+b20+iQI9BBMBCAAnAhsDBQsJCAcDBRUKCQgLBRYCAwEAAh4BAheABQJZHNOBBQkU
+SgzvAAoJEJaz7l8pERFF6xUP/3SbcmrI/u7a2EqZ0GxwQ/LRkPzWkJRnozCtNYHD
+ZjiZgSB/+77hkPS0tsBK/GXFLKfJAuf13XFrCvEuI4Q/pLOCCKIGumKXItUIwJBD
+HiEmVt/XxIijmlF7O1jcWqE/5CQXofjr03WMx+qzNabIwU/6dTKZN4FrR1jDk7yS
+6FYBsbhVcSoqSpGYx7EcuK3c3sKKtnbacK2Sw3K9n8Wdj+EK83cbpMg8D/efVRqv
+xypeCeojtY10y4bmugEwMYPgFkrSbicuiZc8NA8qhvFp6JFRq/uL0PGACyg05wB3
+S9U4wvSkmlo2/G74awna22UlaoYmSSz3UZdpWd2zBxflx17948QfTqyhO6bM8qLz
+dSyR6/6olAcR1N+PBup8PoMdBte4ul/hJp8WIviW0AxJUTZSbVj5v/t43QAKEpCE
+IMHvkK8PRHz/9kMd/2xN7LgMtihCrGZOnzErkjhlZvmiJ6kcJoD7ywzFnfJrntOU
+DjNb3eqUFSEwmhD60Hd2OCkfmiV7NEE/YTd9B72NSwzj4Za/JUdlF64LMeIiHbYp
+Lh7P+mR+lMJf/SWsQmlyuiQ2u8SY2aDFvzBS9WtpwiznuUdrbRN87+TYLSVqDifj
+Ea3zOnzLaLYbOr6LHz1xbhAvInv7KLobgiw1E4WnBNWN8xVwVJLKNE7wV88k43XV
+3L/RuQINBFeeyYwBEADD1Y3zW5OrnYZ6ghTd5PXDAMB8Z1ienmnb2IUzLM+i0yE2
+TpKSP/XYCTBhFa390rYgFO2lbLDVsiz7Txd94nHrdWXGEQfwrbxsvdlLLWk7iN8l
+Fb4B60OfRi3yoR96a/kIPNa0x26+n79LtDuWZ/DTq5JSHztdd9F1sr3h8i5zYmtv
+luj99ZorpwYejbBVUm0+gP0ioaXM37uO56UFVQk3po9GaS+GtLnlgoE5volgNYyO
+rkeIua4uZVsifREkHCKoLJip6P7S3kTyfrpiSLhouEZ7kV1lbMbFgvHXyjm+/AIx
+HIBy+H+e+HNt5gZzTKUJsuBjx44+4jYsOR67EjOdtPOpgiuJXhedzShEO6rbu/O4
+wM1rX45ZXDYa2FGblHCQ/VaS0ttFtztk91xwlWvjTR8vGvp5tIfCi+1GixPRQpbN
+Y/oq8Kv4A7vB3JlJscJCljvRgaX0gTBzlaF6Gq0FdcWEl5F1zvsWCSc/Fv5WrUPY
+5mG0m69YUTeVO6cZS1aiu9Qh3QAT/7NbUuGXIaAxKnu+kkjLSz+nTTlOyvbG7BVF
+a6sDmv48Wqicebkc/rCtO4g8lO7KoA2xC/K/6PAxDrLkVyw8WPsAendmezNfHU+V
+32pvWoQoQqu8ysoaEYc/j9fN4H3mEBCN3QUJYCugmHP0pu7VtpWwwMUqcGeUVwAR
+AQABiQIlBBgBCAAPAhsMBQJZHNOaBQkUSg0HAAoJEJaz7l8pERFFhpkQAJ09mjjp
+n9f18JGSMzP41fVucPuLBZ5XJL/hy2boII1FvgfmOETzNxLPblHdkJVjZS5iMrhL
+EJ1jv+GQDtf68/0jO+HXuQIBmUJ53YwbuuQlLWH7CI2AxlSAKAn2kOApWMKsjnAv
+JwS3eNGukOKWRfEKTqz2Vwi1H7M7ppypZ9keoyAoSIWb61gm7rXbfT+tVBetHfrU
+EM5vz3AS3pJk6Yfqn10IZfiexXmsBD+SpJBNzMBsznCcWO2y4qZNLjFferBoizvV
+34UnZyd1bkSN0T/MKp8sgJwqDJBS72tH6ZIM8NNoy29aPDkeaa8XlhkWiBdRizqL
+BcxrV/1n3xdzfY9FX6s4KGudo+gYsVpY0mrpZU8jG8YUNLDXQTXnRo4CQOtRJJbA
+RFDoZfsDqToZftuEhIsk+MaKlyXoA0eIYqGe6lXa/jEwvViqLYubCNLu0+kgNQ3v
+hKF8Pf7eXFDAePw7guuvDvBOMQqBCaKCxsz1HoKRNYBEdUYrEQBJnX235Q4IsdI/
+GcQ/dvERJXaDCG8EPhnwc517EMUJDiJ1CxT4+VMHphmFbiVqmctz0upIj+D037Xk
+CcgxNte6LZorGRZ/l1MYINliGJKtCCFK7XGVPKiJ8zyGSyPj1FfwtBy5hUX3aQtm
+bvP0H2BRCKoelsbRENu58BkU6YhiUry7pVul
+=SJij
+-----END PGP PUBLIC KEY BLOCK-----

salt/repo/client/files/centos/keys/RPM-GPG-KEY-EPEL-7

@@ -0,0 +1,29 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+Version: GnuPG v1.4.11 (GNU/Linux)
+
+mQINBFKuaIQBEAC1UphXwMqCAarPUH/ZsOFslabeTVO2pDk5YnO96f+rgZB7xArB
+OSeQk7B90iqSJ85/c72OAn4OXYvT63gfCeXpJs5M7emXkPsNQWWSju99lW+AqSNm
+jYWhmRlLRGl0OO7gIwj776dIXvcMNFlzSPj00N2xAqjMbjlnV2n2abAE5gq6VpqP
+vFXVyfrVa/ualogDVmf6h2t4Rdpifq8qTHsHFU3xpCz+T6/dGWKGQ42ZQfTaLnDM
+jToAsmY0AyevkIbX6iZVtzGvanYpPcWW4X0RDPcpqfFNZk643xI4lsZ+Y2Er9Yu5
+S/8x0ly+tmmIokaE0wwbdUu740YTZjCesroYWiRg5zuQ2xfKxJoV5E+Eh+tYwGDJ
+n6HfWhRgnudRRwvuJ45ztYVtKulKw8QQpd2STWrcQQDJaRWmnMooX/PATTjCBExB
+9dkz38Druvk7IkHMtsIqlkAOQMdsX1d3Tov6BE2XDjIG0zFxLduJGbVwc/6rIc95
+T055j36Ez0HrjxdpTGOOHxRqMK5m9flFbaxxtDnS7w77WqzW7HjFrD0VeTx2vnjj
+GqchHEQpfDpFOzb8LTFhgYidyRNUflQY35WLOzLNV+pV3eQ3Jg11UFwelSNLqfQf
+uFRGc+zcwkNjHh5yPvm9odR1BIfqJ6sKGPGbtPNXo7ERMRypWyRz0zi0twARAQAB
+tChGZWRvcmEgRVBFTCAoNykgPGVwZWxAZmVkb3JhcHJvamVjdC5vcmc+iQI4BBMB
+AgAiBQJSrmiEAhsPBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRBqL66iNSxk
+5cfGD/4spqpsTjtDM7qpytKLHKruZtvuWiqt5RfvT9ww9GUUFMZ4ZZGX4nUXg49q
+ixDLayWR8ddG/s5kyOi3C0uX/6inzaYyRg+Bh70brqKUK14F1BrrPi29eaKfG+Gu
+MFtXdBG2a7OtPmw3yuKmq9Epv6B0mP6E5KSdvSRSqJWtGcA6wRS/wDzXJENHp5re
+9Ism3CYydpy0GLRA5wo4fPB5uLdUhLEUDvh2KK//fMjja3o0L+SNz8N0aDZyn5Ax
+CU9RB3EHcTecFgoy5umRj99BZrebR1NO+4gBrivIfdvD4fJNfNBHXwhSH9ACGCNv
+HnXVjHQF9iHWApKkRIeh8Fr2n5dtfJEF7SEX8GbX7FbsWo29kXMrVgNqHNyDnfAB
+VoPubgQdtJZJkVZAkaHrMu8AytwT62Q4eNqmJI1aWbZQNI5jWYqc6RKuCK6/F99q
+thFT9gJO17+yRuL6Uv2/vgzVR1RGdwVLKwlUjGPAjYflpCQwWMAASxiv9uPyYPHc
+ErSrbRG0wjIfAR3vus1OSOx3xZHZpXFfmQTsDP7zVROLzV98R3JwFAxJ4/xqeON4
+vCPFU6OsT3lWQ8w7il5ohY95wmujfr6lk89kEzJdOTzcn7DBbUru33CQMGKZ3Evt
+RjsC7FDbL017qxS+ZVA/HGkyfiu4cpgV8VUnbql5eAZ+1Ll6Dw==
+=hdPa
+-----END PGP PUBLIC KEY BLOCK-----

salt/repo/client/files/centos/keys/SALTSTACK-GPG-KEY.pub

@@ -0,0 +1,31 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+Version: GnuPG v2
+
+mQENBFOpvpgBCADkP656H41i8fpplEEB8IeLhugyC2rTEwwSclb8tQNYtUiGdna9
+m38kb0OS2DDrEdtdQb2hWCnswxaAkUunb2qq18vd3dBvlnI+C4/xu5ksZZkRj+fW
+tArNR18V+2jkwcG26m8AxIrT+m4M6/bgnSfHTBtT5adNfVcTHqiT1JtCbQcXmwVw
+WbqS6v/LhcsBE//SHne4uBCK/GHxZHhQ5jz5h+3vWeV4gvxS3Xu6v1IlIpLDwUts
+kT1DumfynYnnZmWTGc6SYyIFXTPJLtnoWDb9OBdWgZxXfHEcBsKGha+bXO+m2tHA
+gNneN9i5f8oNxo5njrL8jkCckOpNpng18BKXABEBAAG0MlNhbHRTdGFjayBQYWNr
+YWdpbmcgVGVhbSA8cGFja2FnaW5nQHNhbHRzdGFjay5jb20+iQE4BBMBAgAiBQJT
+qb6YAhsDBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRAOCKFJ3le/vhkqB/0Q
+WzELZf4d87WApzolLG+zpsJKtt/ueXL1W1KA7JILhXB1uyvVORt8uA9FjmE083o1
+yE66wCya7V8hjNn2lkLXboOUd1UTErlRg1GYbIt++VPscTxHxwpjDGxDB1/fiX2o
+nK5SEpuj4IeIPJVE/uLNAwZyfX8DArLVJ5h8lknwiHlQLGlnOu9ulEAejwAKt9CU
+4oYTszYM4xrbtjB/fR+mPnYh2fBoQO4d/NQiejIEyd9IEEMd/03AJQBuMux62tjA
+/NwvQ9eqNgLw9NisFNHRWtP4jhAOsshv1WW+zPzu3ozoO+lLHixUIz7fqRk38q8Q
+9oNR31KvrkSNrFbA3D89uQENBFOpvpgBCADJ79iH10AfAfpTBEQwa6vzUI3Eltqb
+9aZ0xbZV8V/8pnuU7rqM7Z+nJgldibFk4gFG2bHCG1C5aEH/FmcOMvTKDhJSFQUx
+uhgxttMArXm2c22OSy1hpsnVG68G32Nag/QFEJ++3hNnbyGZpHnPiYgej3FrerQJ
+zv456wIsxRDMvJ1NZQB3twoCqwapC6FJE2hukSdWB5yCYpWlZJXBKzlYz/gwD/Fr
+GL578WrLhKw3UvnJmlpqQaDKwmV2s7MsoZogC6wkHE92kGPG2GmoRD3ALjmCvN1E
+PsIsQGnwpcXsRpYVCoW7e2nW4wUf7IkFZ94yOCmUq6WreWI4NggRcFC5ABEBAAGJ
+AR8EGAECAAkFAlOpvpgCGwwACgkQDgihSd5Xv74/NggA08kEdBkiWWwJZUZEy7cK
+WWcgjnRuOHd4rPeT+vQbOWGu6x4bxuVf9aTiYkf7ZjVF2lPn97EXOEGFWPZeZbH4
+vdRFH9jMtP+rrLt6+3c9j0M8SIJYwBL1+CNpEC/BuHj/Ra/cmnG5ZNhYebm76h5f
+T9iPW9fFww36FzFka4VPlvA4oB7ebBtquFg3sdQNU/MmTVV4jPFWXxh4oRDDR+8N
+1bcPnbB11b5ary99F/mqr7RgQ+YFF0uKRE3SKa7a+6cIuHEZ7Za+zhPaQlzAOZlx
+fuBmScum8uQTrEF5+Um5zkwC7EXTdH1co/+/V/fpOtxIg4XO4kcugZefVm5ERfVS
+MA==
+=dtMN
+-----END PGP PUBLIC KEY BLOCK-----

salt/repo/client/files/centos/keys/docker.pub

@@ -0,0 +1,28 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+
+mQINBFit5IEBEADDt86QpYKz5flnCsOyZ/fk3WwBKxfDjwHf/GIflo+4GWAXS7wJ
+1PSzPsvSDATV10J44i5WQzh99q+lZvFCVRFiNhRmlmcXG+rk1QmDh3fsCCj9Q/yP
+w8jn3Hx0zDtz8PIB/18ReftYJzUo34COLiHn8WiY20uGCF2pjdPgfxE+K454c4G7
+gKFqVUFYgPug2CS0quaBB5b0rpFUdzTeI5RCStd27nHCpuSDCvRYAfdv+4Y1yiVh
+KKdoe3Smj+RnXeVMgDxtH9FJibZ3DK7WnMN2yeob6VqXox+FvKYJCCLkbQgQmE50
+uVK0uN71A1mQDcTRKQ2q3fFGlMTqJbbzr3LwnCBE6hV0a36t+DABtZTmz5O69xdJ
+WGdBeePCnWVqtDb/BdEYz7hPKskcZBarygCCe2Xi7sZieoFZuq6ltPoCsdfEdfbO
++VBVKJnExqNZCcFUTEnbH4CldWROOzMS8BGUlkGpa59Sl1t0QcmWlw1EbkeMQNrN
+spdR8lobcdNS9bpAJQqSHRZh3cAM9mA3Yq/bssUS/P2quRXLjJ9mIv3dky9C3udM
++q2unvnbNpPtIUly76FJ3s8g8sHeOnmYcKqNGqHq2Q3kMdA2eIbI0MqfOIo2+Xk0
+rNt3ctq3g+cQiorcN3rdHPsTRSAcp+NCz1QF9TwXYtH1XV24A6QMO0+CZwARAQAB
+tCtEb2NrZXIgUmVsZWFzZSAoQ0UgcnBtKSA8ZG9ja2VyQGRvY2tlci5jb20+iQI3
+BBMBCgAhBQJYrep4AhsvBQsJCAcDBRUKCQgLBRYCAwEAAh4BAheAAAoJEMUv62ti
+Hp816C0P/iP+1uhSa6Qq3TIc5sIFE5JHxOO6y0R97cUdAmCbEqBiJHUPNQDQaaRG
+VYBm0K013Q1gcJeUJvS32gthmIvhkstw7KTodwOM8Kl11CCqZ07NPFef1b2SaJ7l
+TYpyUsT9+e343ph+O4C1oUQw6flaAJe+8ATCmI/4KxfhIjD2a/Q1voR5tUIxfexC
+/LZTx05gyf2mAgEWlRm/cGTStNfqDN1uoKMlV+WFuB1j2oTUuO1/dr8mL+FgZAM3
+ntWFo9gQCllNV9ahYOON2gkoZoNuPUnHsf4Bj6BQJnIXbAhMk9H2sZzwUi9bgObZ
+XO8+OrP4D4B9kCAKqqaQqA+O46LzO2vhN74lm/Fy6PumHuviqDBdN+HgtRPMUuao
+xnuVJSvBu9sPdgT/pR1N9u/KnfAnnLtR6g+fx4mWz+ts/riB/KRHzXd+44jGKZra
+IhTMfniguMJNsyEOO0AN8Tqcl0eRBxcOArcri7xu8HFvvl+e+ILymu4buusbYEVL
+GBkYP5YMmScfKn+jnDVN4mWoN1Bq2yMhMGx6PA3hOvzPNsUoYy2BwDxNZyflzuAi
+g59mgJm2NXtzNbSRJbMamKpQ69mzLWGdFNsRd4aH7PT7uPAURaf7B5BVp3UyjERW
+5alSGnBqsZmvlRnVH5BDUhYsWZMPRQS9rRr4iGW0l+TH+O2VJ8aQ
+=0Zqq
+-----END PGP PUBLIC KEY BLOCK-----

salt/repo/client/files/centos/keys/securityonion.pub

@@ -0,0 +1,52 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+
+mQINBF7rzwEBEADBg87uJhnC3Ls7s60hbHGaywGrPtbz2WuYA/ev3YS3X7WS75p8
+PGlzTWUCujx0pEHbK2vYfExl3zksZ8ZmLyZ9VB3oSLiWBzJgKAeB7YCFEo8te+eE
+P2Z+8c+kX4eOV+2waxZyewA2TipSkhWgStSI4Ow8SyVUcUWA3hCw7mo2duNVi7KO
+C3vvI3wzirH+8/XIGo+lWTg6yYlSxdf+0xWzYvV2QCMpwzJfARw6GGXtfCZw/zoO
+o4+YPsiyztQdyI1y+g3Fbesl65E36DelbyP+lYd2VecX8ELEv0wlKCgHYlk6lc+n
+qnOotVjWbsyXuFfo06PHUd6O9n3nmo0drC6kmXGw1e8hu0t8VcGfMTKS/hszwVUY
+bHS6kbfsOoAb6LXPWKfqxk/BdreLXmcHHz88DimS3OS0JufkcmkjxEzSFRL0kb2h
+QVb1SATrbx+v2RWQXvi9sLCjT2fdOiwi1Tgc84orc7A1C3Jwu353YaX9cV+n5uyG
+OZ2AULZ5z2h13sVuiZAwfyyFs/O0CJ783hFA2TNPnyNGAgw/kaIo7nNRnggtndBo
+oQzVS+BHiFx98IF4zDqmF2r2+jOCjxSrw8KnZBe4bgXFtl89DmjoejGvWDnu2MVM
+pZDEs1DcOxHBQmTCWMIYLyNKG0xW6diyWBxEIaa7YgrP6kA+RaDfZ/xXPwARAQAB
+tD9TZWN1cml0eSBPbmlvbiBTb2x1dGlvbnMsIExMQyA8aW5mb0BzZWN1cml0eW9u
+aW9uc29sdXRpb25zLmNvbT6JAlQEEwEKAD4WIQTIBKk9Nr4Mcz6hlkR8EGC3/lBw
+EwUCXuvPAQIbAwUJEswDAAULCQgHAgYVCgkICwIEFgIDAQIeAQIXgAAKCRB8EGC3
+/lBwExB1D/42xIDGU2XFNFyTU+ZqzDA8qNC9hEKjLeizbeM8RIm3xO+3p7SdqbuJ
+7pA8gk0RiHuILb+Ba1xiSh/w/W2bOxQhsXuWHih2z3W1tI+hu6RQhIm4e6CIHHf7
+Vzj4RSvHOVS0AzITUwkHjv0x0Z8zVBPJfEHKkK2x03BqP1o12rd7n2ZMrSfN6sED
+fUwOJLDjthShtyLSPBVG8j7T5cfSCPSLhfVOKPQVcI1sSir7RLeyxt1v1kzjQdaA
++znxO8EgfZJN93wzfBrAGcVT8KmpmgwR6p46m20wJXyZC9DZxJ0o1y3toVWTC+kP
+Qj1ROPivySVn10rBoOJk8HteyhW07gTcydq+noKHV7SqJ1899xRAYP7rDCfI9iMW
+Nn22ZDLnAkIcbNR7JLJCHwsZH/Umo9KO/dIccIqVQel3UCCYZcWTZW0VkcjqVKRa
+eK+JQGaJPrBAoxIG5/sMlbk2sINSubNWlcbH6kM0V8NVwdPiOO9xLmp2hI4ICxE3
+M+O2HCNX4QYzVizzTFxEvW3ieLa4nePQ8J6lvMI2oLkFP7xHoFluvZnuwfNvoEy0
+RnlHExN1UQTUvcbCxIbzjaJ4HJXilWHjgmGaVQO1S7AYskWnNWQ7uJvxnuZBNNwm
+pIvwYEZp23fYaWl/xKqnmPMy2ADjROBKlCm7L+Ntq1r7ELGW5ZCTobkCDQRe688B
+ARAA22GzdkSAo+mwJ2S1RbJ1G20tFnLsG/NC8iMN3lEh/PSmyPdB7mBtjZ+HPDzF
+VSznXZdr3LItBBQOli2hVIj1lZBY7+s2ZufV3TFFwselUwT3b1g1KMkopD95Ckf8
+WhLbSz2yqgrvcEvbB0HFX/ZEsHGqIz2kLacixjwXXLWOMQ2LNbeW1f5zQkBnaNNQ
+/4njzTj68OxnvfplNYNJqi2pZGb2UqarYX04FqKNuocN8E7AC9FQdBXylmVctw9T
+pQVwfCI76bTe6vPWb+keb6UNN1jyXVnhIQ3Fv5sFBsmgXf/hO8tqCotrKjEiK2/i
+RkvFeqsGMXreCgYg9zW4k+DcJtVa+Q8juGOjElrubY3Ua9mCusx3vY4QYSWxQ5Ih
+k1lXiUcM5Rt38lfpKHRJ5Pd4Y5xlWSQfZ7nmzbf/GzJQz+rWrA0X6Oc6cDOPLNXK
+w1dAygre4f2bsp5kHQt6NMefxeNTDmi+4R62K0tb40f5q0Vxz8qdyD48bBsbULNx
+kb6mjOAD+FNkfNXcGeuTq9oRnjx8i93mhYsIP5LFNDXS/zSP1nv0ZUFeIlGQGjV9
+1wOvT454qkI9sKiVFtd4FrNKZJbKszxxDm+DPfB5j+hRC4oeEJ7w+sVyh3EawtfM
+V7Mwj8i+7c3YUCravXBhSwG7SCTggFUgA8lMr8oWVgCATYsAEQEAAYkCPAQYAQoA
+JhYhBMgEqT02vgxzPqGWRHwQYLf+UHATBQJe688BAhsMBQkSzAMAAAoJEHwQYLf+
+UHATTtwQAJiztPW68ykifpFdwYFp1VC7c+uGLhWBqjDY9NSUKNC9caR7bV0cnNu8
+07UG6j18gCB2GSkukXjOR/oTj6rNcW/WouPYfQOrw7+M2Ya8M8iq+E/HOXaXB3b4
+FeCcB0UuwfcHHd2KbXrRHA+9GNpmuOcfTCdsPpIr41Xg4QltATDEt/FrzuKspXg4
+vUKDXgfnbj7y0JcJM2FfcwWGlnAG5MMRyjJQAleGdiidX/9WxgJ4Mweq4qJM0jr3
+Qsrc9VuzxsLr85no3Hn5UYVgT7bBZ59HUbQoi775m78MxN3mWUSdcyLQKovI+YXr
+tshTxWIf/2Ovdzt6Wq1WWXOGGuK1qgdPJTFWrlh3amFdb70zR1p6A/Lthd7Zty+n
+QjRZRQo5jBSnYtjhMrZP6rxM3QqnQ0frEKK9HfDYONk1Bw18CUtdwFGb9OMregLR
+IjvNLp9coSh5yYAepZyUGEPRET0GsmVw2trQF0uyMSkQfiq2zjPto6WWbsmrrbLr
+cfZ/wnBw1FoNEd51U54euo9yvOgOVtJGvqLgHNwB8574FhQhoWAMhyizqdgeEt26
+m3FXecUNKL/AK71/l04vor+/WsXe8uhDg3O84qeYa9wgd8LZZVmGZJDosSwqYjtb
+LdNNm+v60Zo6rFWSREegqi/nRTTDdxdW99ybjlh+mpbq3xavyFXF
+=bhkm
+-----END PGP PUBLIC KEY BLOCK-----

salt/repo/client/files/centos/securityonion.repo

@@ -0,0 +1,78 @@
+[base]
+name=CentOS-$releasever - Base
+baseurl=https://repo.securityonion.net/file/securityonion-repo/base/
+gpgcheck=1
+gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
+
+#released updates 
+[updates]
+name=CentOS-$releasever - Updates
+baseurl=https://repo.securityonion.net/file/securityonion-repo/updates/
+gpgcheck=1
+gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
+
+#additional packages that may be useful
+[extras]
+name=CentOS-$releasever - Extras
+baseurl=https://repo.securityonion.net/file/securityonion-repo/extras/
+gpgcheck=1
+gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
+
+#additional packages that extend functionality of existing packages
+[centosplus]
+name=CentOS-$releasever - Plus
+baseurl=https://repo.securityonion.net/file/securityonion-repo/centosplus/
+gpgcheck=1
+enabled=0
+gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
+
+[epel]
+name=Extra Packages for Enterprise Linux 7 - $basearch
+baseurl=https://repo.securityonion.net/file/securityonion-repo/epel/
+enabled=1
+gpgcheck=1
+gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL-7
+
+[docker-ce-stable]
+name=Docker CE Stable - $basearch
+baseurl=https://repo.securityonion.net/file/securityonion-repo/docker-ce-stable
+enabled=1
+gpgcheck=1
+gpgkey=file:///etc/pki/rpm-gpg/docker.pub
+
+[saltstack]
+name=SaltStack repo for RHEL/CentOS $releasever PY3
+baseurl=https://repo.securityonion.net/file/securityonion-repo/saltstack3003/
+enabled=1
+gpgcheck=1
+gpgkey=file:///etc/pki/rpm-gpg/SALTSTACK-GPG-KEY.pub
+
+[saltstack3003]
+name=SaltStack repo for RHEL/CentOS $releasever PY3
+baseurl=https://repo.securityonion.net/file/securityonion-repo/saltstack3003/
+enabled=1
+gpgcheck=1
+gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/SALTSTACK-GPG-KEY.pub
+
+[wazuh_repo]
+gpgcheck=1
+gpgkey=file:///etc/pki/rpm-gpg/GPG-KEY-WAZUH
+enabled=1
+name=Wazuh repository
+baseurl=https://repo.securityonion.net/file/securityonion-repo/wazuh_repo/
+protect=1
+
+[wazuh4_repo]
+gpgcheck=1
+gpgkey=file:///etc/pki/rpm-gpg/GPG-KEY-WAZUH
+enabled=1
+name=Wazuh repository
+baseurl=https://repo.securityonion.net/file/securityonion-repo/wazuh4_repo/
+protect=1
+
+[securityonion]
+name=Security Onion Repo repo
+baseurl=https://repo.securityonion.net/file/securityonion-repo/securityonion/
+enabled=1
+gpgcheck=1
+gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/securityonion.pub

salt/repo/client/files/centos/securityonioncache.repo

@@ -0,0 +1,78 @@
+[base]
+name=CentOS-$releasever - Base
+baseurl=http://repocache.securityonion.net/file/securityonion-repo/base/
+gpgcheck=1
+gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
+
+#released updates 
+[updates]
+name=CentOS-$releasever - Updates
+baseurl=http://repocache.securityonion.net/file/securityonion-repo/updates/
+gpgcheck=1
+gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
+
+#additional packages that may be useful
+[extras]
+name=CentOS-$releasever - Extras
+baseurl=http://repocache.securityonion.net/file/securityonion-repo/extras/
+gpgcheck=1
+gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
+
+#additional packages that extend functionality of existing packages
+[centosplus]
+name=CentOS-$releasever - Plus
+baseurl=http://repocache.securityonion.net/file/securityonion-repo/centosplus/
+gpgcheck=1
+enabled=0
+gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
+
+[epel]
+name=Extra Packages for Enterprise Linux 7 - $basearch
+baseurl=http://repocache.securityonion.net/file/securityonion-repo/epel/
+enabled=1
+gpgcheck=1
+gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/RPM-GPG-KEY-EPEL-7
+
+[docker-ce-stable]
+name=Docker CE Stable - $basearch
+baseurl=http://repocache.securityonion.net/file/securityonion-repo/docker-ce-stable
+enabled=1
+gpgcheck=1
+gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/docker.pub
+
+[saltstack]
+name=SaltStack repo for RHEL/CentOS $releasever PY3
+baseurl=http://repocache.securityonion.net/file/securityonion-repo/saltstack3003/
+enabled=1
+gpgcheck=1
+gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/SALTSTACK-GPG-KEY.pub
+
+[saltstack3003]
+name=SaltStack repo for RHEL/CentOS $releasever PY3
+baseurl=http://repocache.securityonion.net/file/securityonion-repo/saltstack3003/
+enabled=1
+gpgcheck=1
+gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/SALTSTACK-GPG-KEY.pub
+
+[wazuh_repo]
+gpgcheck=1
+gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/GPG-KEY-WAZUH
+enabled=1
+name=Wazuh repository
+baseurl=http://repocache.securityonion.net/file/securityonion-repo/wazuh_repo/
+protect=1
+
+[wazuh4_repo]
+gpgcheck=1
+gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/GPG-KEY-WAZUH
+enabled=1
+name=Wazuh repository
+baseurl=http://repocache.securityonion.net/file/securityonion-repo/wazuh4_repo/
+protect=1
+
+[securityonion]
+name=Security Onion Repo
+baseurl=http://repocache.securityonion.net/file/securityonion-repo/securityonion/
+enabled=1
+gpgcheck=1
+gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/securityonion.pub

salt/repo/client/files/centos/yum.conf.jinja

@@ -12,7 +12,7 @@ installonly_limit={{ salt['pillar.get']('yum:config:installonly_limit', 2) }}
 bugtracker_url=http://bugs.centos.org/set_project.php?project_id=23&ref=http://bugs.centos.org/bug_report_page.php?category=yum
 distroverpkg=centos-release
 clean_requirements_on_remove=1
-{% if (grains['role'] not in ['so-eval','so-managersearch', 'so-manager', 'so-standalone']) and salt['pillar.get']('global:managerupdate', '0') -%}
+{% if (grains['role'] not in ['so-eval','so-managersearch', 'so-manager', 'so-standalone', 'so-import']) and ( salt['pillar.get']('global:managerupdate', '0') or salt['pillar.get']('patch:os:source', 'direct') == 'manager' ) -%}
 proxy=http://{{ salt['pillar.get']('yum:config:proxy', salt['config.get']('master')) }}:3142
 {% elif proxy -%}
 proxy={{ proxy }}

salt/repo/client/init.sls

@@ -0,0 +1,83 @@
+{% from 'repo/client/map.jinja' import ABSENTFILES with context %}
+{% from 'repo/client/map.jinja' import REPOPATH with context %}
+{% set ISAIRGAP = salt['pillar.get']('global:airgap', False) %}
+{% set managerupdates = salt['pillar.get']('global:managerupdate', '0') %}
+{% set role = grains.id.split('_') | last %}
+
+# from airgap state
+{% if ISAIRGAP and grains.os == 'CentOS' %}
+{% set MANAGER = salt['grains.get']('master') %}
+airgapyum:
+  file.managed:
+    - name: /etc/yum/yum.conf
+    - source: salt://repo/client/files/centos/airgap/yum.conf
+
+airgap_repo:
+  pkgrepo.managed:
+    - humanname: Airgap Repo
+    - baseurl: https://{{ MANAGER }}/repo
+    - gpgcheck: 0
+    - sslverify: 0
+
+{% endif %}
+
+# from airgap and common
+{% if ABSENTFILES|length > 0%}
+  {% for file in ABSENTFILES  %}
+{{ file }}:
+  file.absent:
+    - name: {{ REPOPATH }}{{ file }}
+    - onchanges_in:
+      - module: cleanyum
+  {% endfor %}
+{% endif %}
+
+# from common state
+# Remove default Repos
+{% if grains['os'] == 'CentOS' %}
+repair_yumdb:
+  cmd.run:
+    - name: 'mv -f /var/lib/rpm/__db* /tmp && yum clean all'
+    - onlyif:
+      - 'yum check-update 2>&1 | grep "Error: rpmdb open failed"'
+
+crsynckeys:
+  file.recurse:
+    - name: /etc/pki/rpm_gpg
+    - source: salt://repo/client/files/centos/keys/
+
+{% if not ISAIRGAP %}
+crsecurityonionrepo:
+  file.managed:
+    {% if role in ['eval', 'standalone', 'import', 'manager', 'managersearch'] or managerupdates == 0 %}
+    - name: /etc/yum.repos.d/securityonion.repo
+    - source: salt://repo/client/files/centos/securityonion.repo
+    {% else %}
+    - name: /etc/yum.repos.d/securityonioncache.repo
+    - source: salt://repo/client/files/centos/securityonioncache.repo
+    {% endif %}
+    - mode: 644
+
+yumconf:
+  file.managed:
+    - name: /etc/yum.conf
+    - source: salt://repo/client/files/centos/yum.conf.jinja
+    - mode: 644
+    - template: jinja
+    - show_changes: False
+{% endif %}
+
+cleanyum:
+  module.run:
+    - pkg.clean_metadata: []
+    - onchanges:
+{% if ISAIRGAP %}
+      - file: airgapyum
+      - pkgrepo: airgap_repo
+{% else %}
+      - file: crsecurityonionrepo
+      - file: yumconf
+{% endif %}
+
+{% endif %}
+

salt/repo/client/map.jinja

@@ -0,0 +1,26 @@
+{% if grains.os == 'CentOS' %}
+
+  {% set REPOPATH = '/etc/yum.repos.d/' %}
+  {% set ABSENTFILES = [
+         'CentOS-Base.repo',
+         'CentOS-CR.repo',
+         'CentOS-Debuginfo.repo',
+         'CentOS-fasttrack.repo',
+         'CentOS-Media.repo',
+         'CentOS-Sources.repo',
+         'CentOS-Vault.repo',
+         'CentOS-x86_64-kernel.repo',
+         'docker-ce.repo',
+         'epel.repo',
+         'epel-testing.repo',
+         'saltstack.repo',
+         'wazuh.repo'
+       ]
+  %}
+
+{% elif grains.os == 'Ubuntu' %}
+
+  {% set REPOPATH = '/etc/apt/sources.list.d/' %}
+  {% set ABSENTFILES = [] %}
+
+{% endif %}

salt/salt/map.jinja

@@ -3,28 +3,21 @@
 
 {% if grains.os == 'Ubuntu' %}
   {% set SPLITCHAR = '+' %}
+  {% set SALTNOTHELD = salt['cmd.run']('apt-mark showhold | grep -q salt ; echo $?', python_shell=True) %}
+  {% set SALTPACKAGES = ['salt-common', 'salt-master', 'salt-minion'] %}
 {% else %}
   {% set SPLITCHAR = '-' %}
+  {% set SALTNOTHELD = salt['cmd.run']('yum versionlock list | grep -q salt ; echo $?', python_shell=True) %}
+  {% set SALTPACKAGES = ['salt', 'salt-master', 'salt-minion'] %}
 {% endif %}
 
 {% set INSTALLEDSALTVERSION = salt['pkg.version']('salt-minion').split(SPLITCHAR)[0] %}
-{% set ISAIRGAP = salt['pillar.get']('global:airgap', 'False') %}
-
-{% if grains.os|lower == 'ubuntu' %}
-  {% set COMMON = 'salt-common' %}
-{% elif grains.os|lower in ['centos', 'redhat'] %}
-  {% set COMMON = 'salt' %}
-{% endif %}
 
 {% if grains.saltversion|string != SALTVERSION|string %}
   {% if grains.os|lower in ['centos', 'redhat'] %}
-    {% if ISAIRGAP is sameas true %}
-      {% set UPGRADECOMMAND = 'yum clean all && yum versionlock delete "salt-*" && /usr/sbin/bootstrap-salt.sh -X -s 120 -r -F -x python3 stable ' ~ SALTVERSION ~ ' && yum versionlock add "salt-*"' %}
-    {% else %}
-      {% set UPGRADECOMMAND = 'yum versionlock delete "salt-*" && /usr/sbin/bootstrap-salt.sh -X -s 120 -F -x python3 stable ' ~ SALTVERSION ~ ' && yum versionlock add "salt-*"' %}
-    {% endif %}
+      {% set UPGRADECOMMAND = 'yum clean all ; /usr/sbin/bootstrap-salt.sh -X -s 120 -r -F -x python3 stable ' ~ SALTVERSION %}
   {% elif grains.os|lower == 'ubuntu' %}
-    {% set UPGRADECOMMAND = 'apt-mark unhold salt-common && apt-mark unhold salt-minion && /usr/sbin/bootstrap-salt.sh -X -s 120 -F -x python3 stable ' ~ SALTVERSION ~ ' && apt-mark hold salt-common && apt-mark hold salt-minion' %}
+    {% set UPGRADECOMMAND = '/usr/sbin/bootstrap-salt.sh -X -s 120 -F -x python3 stable ' ~ SALTVERSION %}
   {% endif %}
 {% else %}
   {% set UPGRADECOMMAND = 'echo Already running Salt Minion version ' ~ SALTVERSION %}

salt/salt/master.defaults.yaml

@@ -2,4 +2,4 @@
 # When updating the salt version, also update the version in securityonion-builds/images/iso-task/Dockerfile and saltify function in so-functions
 salt:
   master:
-    version: 3002.5
+    version: 3003

salt/salt/master.sls

@@ -1,17 +1,16 @@
+{% from 'salt/map.jinja' import SALTNOTHELD %}
 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% if sls in allowed_states %}
 
-{% from 'salt/map.jinja' import COMMON with context %}
-
 include:
   - salt.minion
 
-salt_master_package:
-  pkg.installed:
-    - pkgs:
-      - {{ COMMON }}
-      - salt-master
-    - hold: True
+{% if SALTNOTHELD == 1 %}
+hold_salt_master_package:
+  module.run:
+    - pkg.hold:
+      - name: salt-master
+{% endif %}
 
 salt_master_service:
   service.running:

salt/salt/minion.defaults.yaml

@@ -2,5 +2,6 @@
 # When updating the salt version, also update the version in securityonion-builds/images/iso-task/Dockerfile and saltify function in so-functions
 salt:
   minion:
-    version: 3002.5
+    version: 3003
     check_threshold: 3600 # in seconds, threshold used for so-salt-minion-check. any value less than 600 seconds may cause a lot of salt-minion restarts since the job to touch the file occurs every 5-8 minutes by default
+    service_start_delay: 30 # in seconds.

salt/salt/minion.sls

@@ -1,10 +1,26 @@
-{% from 'salt/map.jinja' import COMMON with context %}
 {% from 'salt/map.jinja' import UPGRADECOMMAND with context %}
 {% from 'salt/map.jinja' import SALTVERSION %}
 {% from 'salt/map.jinja' import INSTALLEDSALTVERSION %}
+{% from 'salt/map.jinja' import SALTNOTHELD %}
+{% from 'salt/map.jinja' import SALTPACKAGES %}
+{% import_yaml 'salt/minion.defaults.yaml' as SALTMINION %}
+{% set service_start_delay = SALTMINION.salt.minion.service_start_delay %}
 
 include:
   - salt
+  - systemd.reload
+
+{% if INSTALLEDSALTVERSION|string != SALTVERSION|string %}
+
+{% if SALTNOTHELD | int == 0 %}
+unhold_salt_packages:
+  module.run:
+    - pkg.unhold:
+      - pkgs:
+{% for package in SALTPACKAGES %}
+        - {{ package }}
+{% endfor %}
+{% endif %}
 
 install_salt_minion:
   cmd.run:
@@ -13,25 +29,55 @@ install_salt_minion:
         exec 1>&- # close stdout
         exec 2>&- # close stderr
         nohup /bin/sh -c '{{ UPGRADECOMMAND }}' &
-    - onlyif: test "{{INSTALLEDSALTVERSION}}" != "{{SALTVERSION}}"
+{% endif %}
 
-salt_minion_package:
-  pkg.installed:
+{% if INSTALLEDSALTVERSION|string == SALTVERSION|string %}
+
+{% if SALTNOTHELD | int == 1 %}
+hold_salt_packages:
+  module.run:
+    - pkg.hold:
       - pkgs:
-      - {{ COMMON }}
-      - salt-minion
-    - hold: True
-    - onlyif: test "{{INSTALLEDSALTVERSION}}" == "{{SALTVERSION}}"
+{% for package in SALTPACKAGES %}
+        - {{ package }}
+{% endfor %}
+{% endif %}
+
+remove_info_log_level_logfile:
+  file.line:
+    - name: /etc/salt/minion
+    - match: "log_level_logfile: info"
+    - mode: delete
+
+remove_info_log_level:
+  file.line:
+    - name: /etc/salt/minion
+    - match: "log_level: info"
+    - mode: delete
 
 set_log_levels:
   file.append:
     - name: /etc/salt/minion
     - text:
-      - "log_level: info"
-      - "log_level_logfile: info"
+      - "log_level: error"
+      - "log_level_logfile: error"
     - listen_in:
       - service: salt_minion_service
 
+salt_minion_service_unit_file:
+  file.managed:
+    - name: /etc/systemd/system/multi-user.target.wants/salt-minion.service
+    - source: salt://salt/service/salt-minion.service.jinja
+    - template: jinja
+    - defaults:
+        service_start_delay: {{ service_start_delay }}
+    - onchanges_in:
+      - module: systemd_reload
+    - listen_in:
+      - service: salt_minion_service
+{% endif %}
+
+# this has to be outside the if statement above since there are <requisite>_in calls to this state
 salt_minion_service:
   service.running:
     - name: salt-minion

salt/salt/service/salt-minion.service.jinja

@@ -0,0 +1,15 @@
+[Unit]
+Description=The Salt Minion
+Documentation=man:salt-minion(1) file:///usr/share/doc/salt/html/contents.html https://docs.saltstack.com/en/latest/contents.html
+After=network.target salt-master.service
+
+[Service]
+KillMode=process
+Type=notify
+NotifyAccess=all
+LimitNOFILE=8192
+ExecStart=/usr/bin/salt-minion
+ExecStartPre=/bin/sleep {{ salt['pillar.get']('salt:minion:service_start_delay', service_start_delay) }}
+
+[Install]
+WantedBy=multi-user.target

salt/sensoroni/files/sensoroni.json

@@ -1,5 +1,6 @@
 {%- set URLBASE = salt['pillar.get']('global:url_base') %}
-{%- set DESCRIPTION = salt['pillar.get']('sensoroni:node_description') %}
+{%- set DESCRIPTION = salt['pillar.get']('sensoroni:node_description', '') %}
+{%- set MODEL = salt['grains.get']('sosmodel', '') %}
 {%- set ADDRESS = salt['pillar.get']('sensoroni:node_address') %}
 {%- set SENSORONIKEY = salt['pillar.get']('global:sensoronikey', '') %}
 {%- set CHECKININTERVALMS = salt['pillar.get']('sensoroni:node_checkin_interval_ms', 10000) %}
@@ -14,9 +15,11 @@
   "logFilename": "/opt/sensoroni/logs/sensoroni.log",
   "logLevel":"info",
   "agent": {
+    "nodeId": "{{ grains.host | lower }}",
     "role": "{{ grains.role }}",
     "description": "{{ DESCRIPTION }}",
     "address": "{{ ADDRESS }}",
+    "model": "{{ MODEL }}",
     "pollIntervalMs": {{ CHECKININTERVALMS if CHECKININTERVALMS else 10000 }},
     "serverUrl": "https://{{ URLBASE }}/sensoroniagents",
     "verifyCert": false,

salt/soc/files/soc/alerts.actions.json

@@ -1,7 +1,7 @@
 [
   { "name": "actionHunt", "description": "actionHuntHelp", "icon": "fa-crosshairs", "target": "", 
             "links": [
-              "/#/hunt?q=\"{value}\" | groupby event.module event.dataset"
+              "/#/hunt?q=\"{value|escape}\" | groupby event.module event.dataset"
             ]},
   { "name": "actionCorrelate", "description": "actionCorrelateHelp", "icon": "fab fa-searchengin", "target": "",
             "links": [

salt/soc/files/soc/changes.json

@@ -1,49 +0,0 @@
-{
-  "title": "Security Onion 2.3.40 is here!",
-  "changes": [
-    { "summary": "FEATURE: Add option for HTTP Method Specification/POST to Hunt/Alerts Actions <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/2904\">#2904</a>" },
-    { "summary": "FEATURE: Add option to configure proxy for various tools used during setup + persist the proxy configuration <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/529\">#529</a>" },
-    { "summary": "FEATURE: Alerts/Hunt - Provide method for base64-encoding pivot value <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/1749\">#1749</a>" },
-    { "summary": "FEATURE: Allow users to customize links in SOC <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/1248\">#1248</a>" },
-    { "summary": "FEATURE: Display user who requested PCAP in SOC <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/2775\">#2775</a>" },
-    { "summary": "FEATURE: Make SOC browser app connection timeouts adjustable <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/2408\">#2408</a>" },
-    { "summary": "FEATURE: Move to FleetDM <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/3483\">#3483</a>" },
-    { "summary": "FEATURE: Reduce field cache expiration from 1d to 5m, and expose value as a salt pillar <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/3537\">#3537</a>" },
-    { "summary": "FEATURE: Refactor docker_clean salt state to use loop w/ inspection instead of hardcoded image list <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/3113\">#3113</a>" },
-    { "summary": "FEATURE: Run so-ssh-harden during setup <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/1932\">#1932</a>" },
-    { "summary": "FEATURE: SOC should only display links to tools that are enabled <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/1643\">#1643</a>" },
-    { "summary": "FEATURE: Update Sigmac Osquery Field Mappings <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/3137\">#3137</a>" },
-    { "summary": "FEATURE: User must accept the Elastic licence during setup <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/3233\">#3233</a>" },
-    { "summary": "FEATURE: soup should output more guidance for distributed deployments at the end <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/3340\">#3340</a>" },
-    { "summary": "FEATURE: soup should provide some initial information and then prompt the user to continue <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/3486\">#3486</a>" },
-    { "summary": "FIX: Add cronjob for so-suricata-eve-clean script <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/3515\">#3515</a>" },
-    { "summary": "FIX: Change Elasticsearch heap formula <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/1686\">#1686</a>" },
-    { "summary": "FIX: Create a post install version loop in soup <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/3102\">#3102</a>" },
-    { "summary": "FIX: Custom Kibana settings are not being applied properly on upgrades <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/3254\">#3254</a>" },
-    { "summary": "FIX: Hunt query issues with quotes <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/3320\">#3320</a>" },
-    { "summary": "FIX: IP Addresses don't work with .security <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/3327\">#3327</a>" },
-    { "summary": "FIX: Improve DHCP leases query in Hunt <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/3395\">#3395</a>" },
-    { "summary": "FIX: Improve Setup verbiage <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/3422\">#3422</a>" },
-    { "summary": "FIX: Improve Suricata DHCP logging and parsing <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/3397\">#3397</a>" },
-    { "summary": "FIX: Keep RELATED,ESTABLISHED rules at the top of iptables chains <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/3288\">#3288</a>" },
-    { "summary": "FIX: Populate http.status_message field <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/3408\">#3408</a>" },
-    { "summary": "FIX: Remove 'types removal' deprecation messages from elastic log. <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/3345\">#3345</a>" },
-    { "summary": "FIX: Reword + fix formatting on ES data storage prompt <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/3205\">#3205</a>" },
-    { "summary": "FIX: SMTP shoud read SNMP on Kibana SNMP view <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/3413\">#3413</a>" },
-    { "summary": "FIX: Sensors can temporarily show offline while processing large PCAP jobs <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/3279\">#3279</a>" },
-    { "summary": "FIX: Soup should log to the screen as well as to a file <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/3467\">#3467</a>" },
-    { "summary": "FIX: Strelka port 57314 not immediately relinquished upon restart <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/3457\">#3457</a>" },
-    { "summary": "FIX: Switch SOC to pull from fieldcaps API due to field caching changes in Kibana 7.11 <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/3502\">#3502</a>" },
-    { "summary": "FIX: Syntax error in /etc/sysctl.d/99-reserved-ports.conf <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/3308\">#3308</a>" },
-    { "summary": "FIX: Telegraf hardcoded to use https and is not aware of elasticsearch features <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/2061\">#2061</a>" },
-    { "summary": "FIX: Zeek Index Close and Delete Count for curator <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/3274\">#3274</a>" },
-    { "summary": "FIX: so-cortex-user-add and so-cortex-user-enable use wrong pillar value for api key <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/3388\">#3388</a>" },
-    { "summary": "FIX: so-rule does not completely apply change <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/3289\">#3289</a>" },
-    { "summary": "FIX: soup should recheck disk space after it tries to clean up. <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/3235\">#3235</a>" },
-    { "summary": "UPGRADE: Elastic 7.11.2 <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/3389\">#3389</a>" },
-    { "summary": "UPGRADE: Suricata 6.0.2 <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/3217\">#3217</a>" },
-    { "summary": "UPGRADE: Zeek 4 <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/3216\">#3216</a>" },
-    { "summary": "UPGRADE: Zeek container to use Python 3 <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/1113\">#1113</a>" },
-    { "summary": "UPGRADE: docker-ce to latest <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/3493\">#3493</a>" }
-  ]
-}

salt/soc/files/soc/custom.js

@@ -18,7 +18,4 @@
    content placed here so that upgrading to newer version of 
    Security Onion do not become a burden.
    
-	 Example:
-
-	   i18n.translations["en-US"].loginHeader = "Unauthorized use of this computer system is prohibited...";
 */

salt/soc/files/soc/hunt.actions.json

@@ -1,7 +1,7 @@
 [
   { "name": "actionHunt", "description": "actionHuntHelp", "icon": "fa-crosshairs", "target": "", 
             "links": [
-              "/#/hunt?q=\"{value}\" | groupby event.module event.dataset"
+              "/#/hunt?q=\"{value|escape}\" | groupby event.module event.dataset"
             ]},
   { "name": "actionCorrelate", "description": "actionCorrelateHelp", "icon": "fab fa-searchengin", "target": "",
             "links": [

salt/soc/files/soc/hunt.queries.json

@@ -34,7 +34,7 @@
           { "name": "HTTP", "description": "HTTP grouped by status code and message", "query": "event.dataset:http | groupby http.status_code http.status_message"},
           { "name": "HTTP", "description": "HTTP grouped by method and user agent", "query": "event.dataset:http | groupby http.method http.useragent"},
           { "name": "HTTP", "description": "HTTP grouped by virtual host", "query": "event.dataset:http | groupby http.virtual_host"},
-          { "name": "HTTP", "description": "HTTP with exe downloads", "query": "event.dataset:http AND file.resp_mime_types:dosexec | groupby http.virtual_host"},
+          { "name": "HTTP", "description": "HTTP with exe downloads", "query": "event.dataset:http AND (file.resp_mime_types:dosexec OR file.resp_mime_types:executable) | groupby http.virtual_host"},
           { "name": "Intel", "description": "Intel framework hits grouped by indicator", "query": "event.dataset:intel | groupby intel.indicator.keyword"},
           { "name": "IRC", "description": "IRC grouped by command", "query": "event.dataset:irc | groupby irc.command.type"},
           { "name": "KERBEROS", "description": "KERBEROS grouped by service", "query": "event.dataset:kerberos | groupby kerberos.service"},

salt/soc/files/soc/motd.md

@@ -0,0 +1,27 @@
+## Getting Started
+
+New to Security Onion 2? Check out the [Online Help](/docs/) and [Cheatsheet](/docs/cheatsheet.pdf) to learn how to best utilize Security Onion to hunt for evil! Find them in the upper-right menu. Also, watch our free Security Onion 2 Essentials online course, available on our [Training](https://securityonionsolutions.com/training) website.
+
+If you're ready to dive-in, take a look at the [Alerts](/#/alerts) interface to see what Security Onion has detected so far. Or navigate to the [Hunt](/#/hunt) interface to hunt for evil that the alerts might have missed!
+
+## What's New 
+
+The release notes have moved to the upper-right menu. Click on the [What's New](/docs/#document-release-notes) menu option to find all the latest fixes and features in this version of Security Onion!
+
+## Customize This Space
+
+Make this area your own by customizing the content. The content is stored in the `motd.md` file, which uses the common Markdown (.md) format. Visit [markdownguide.org](https://www.markdownguide.org/) to learn more about the simple Markdown format.
+
+To customize this content, login to the manager via SSH and execute the following command:
+
+```bash
+sudo cp /opt/so/saltstack/default/salt/soc/files/soc/motd.md /opt/so/saltstack/local/salt/soc/files/soc/
+```
+
+and edit the new file as desired. 
+
+Finally, run this command:
+
+```bash
+sudo so-soc-restart
+```

salt/soc/files/soc/soc.json

@@ -53,6 +53,17 @@
         "cacheMs": {{ ES_FIELDCAPS_CACHE }},
         "verifyCert": false
       },
+      "influxdb": {
+{%- if grains['role'] in ['so-import'] or (grains['role'] == 'so-eval' and GRAFANA == 0) %}
+        "hostUrl": "",
+{%- else %}
+        "hostUrl": "https://{{ MANAGERIP }}:8086",
+{%- endif %}
+        "token": "",
+        "org": "",
+        "bucket": "telegraf",
+        "verifyCert": false
+      },
       "sostatus": {
         "refreshIntervalMs": 30000,
         "offlineThresholdMs": 900000

salt/soc/init.sls

@@ -35,10 +35,19 @@ socconfig:
     - mode: 600
     - template: jinja
 
-socchanges:
+socmotd:
   file.managed:
-    - name: /opt/so/conf/soc/changes.json
-    - source: salt://soc/files/soc/changes.json
+    - name: /opt/so/conf/soc/motd.md
+    - source: salt://soc/files/soc/motd.md
+    - user: 939
+    - group: 939
+    - mode: 600
+    - template: jinja
+
+socbanner:
+  file.managed:
+    - name: /opt/so/conf/soc/banner.md
+    - source: salt://soc/files/soc/banner.md
     - user: 939
     - group: 939
     - mode: 600
@@ -61,7 +70,8 @@ so-soc:
     - binds:
       - /nsm/soc/jobs:/opt/sensoroni/jobs:rw
       - /opt/so/conf/soc/soc.json:/opt/sensoroni/sensoroni.json:ro
-      - /opt/so/conf/soc/changes.json:/opt/sensoroni/html/changes.json:ro
+      - /opt/so/conf/soc/motd.md:/opt/sensoroni/html/motd.md:ro
+      - /opt/so/conf/soc/banner.md:/opt/sensoroni/html/login/banner.md:ro
       - /opt/so/conf/soc/custom.js:/opt/sensoroni/html/js/custom.js:ro
       - /opt/so/log/soc/:/opt/sensoroni/logs/:rw
     {%- if salt['pillar.get']('nodestab', {}) %}

salt/soctopus/files/SOCtopus.conf

@@ -8,8 +8,8 @@
 [es]
 es_url = https://{{MANAGER}}:9200
 es_ip = {{MANAGER}}
-es_user = YOURESUSER
-es_pass = YOURESPASS
+es_user = 
+es_pass = 
 es_index_pattern = so-*
 es_verifycert = no
 

salt/ssl/init.sls

@@ -68,8 +68,9 @@ removeesp12dir:
       - x509: /etc/pki/influxdb.crt
     {%- endif %}
     - timeout: 30
-    - retry: 5
-    - interval: 30
+    - retry:
+        attempts: 5
+        interval: 30
 
 # Create a cert for the talking to influxdb
 /etc/pki/influxdb.crt:
@@ -86,8 +87,9 @@ removeesp12dir:
       # Will trigger 5 days (432000 sec) from cert expiration
       - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/influxdb.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]'
     - timeout: 30
-    - retry: 5
-    - interval: 30
+    - retry:
+        attempts: 5
+        interval: 30
 
 influxkeyperms:
   file.managed:
@@ -111,8 +113,9 @@ influxkeyperms:
       - x509: /etc/pki/redis.crt
     {%- endif %}
     - timeout: 30
-    - retry: 5
-    - interval: 30
+    - retry:
+        attempts: 5
+        interval: 30
 
 /etc/pki/redis.crt:
   x509.certificate_managed:
@@ -128,8 +131,9 @@ influxkeyperms:
       # Will trigger 5 days (432000 sec) from cert expiration
       - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/redis.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]'
     - timeout: 30
-    - retry: 5
-    - interval: 30
+    - retry:
+        attempts: 5
+        interval: 30
 
 rediskeyperms:
   file.managed:
@@ -153,8 +157,9 @@ rediskeyperms:
       - x509: /etc/pki/filebeat.crt
     {%- endif %}
     - timeout: 30
-    - retry: 5
-    - interval: 30
+    - retry:
+        attempts: 5
+        interval: 30
 
 # Request a cert and drop it where it needs to go to be distributed
 /etc/pki/filebeat.crt:
@@ -175,8 +180,9 @@ rediskeyperms:
       # Will trigger 5 days (432000 sec) from cert expiration
       - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/filebeat.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]'
     - timeout: 30
-    - retry: 5
-    - interval: 30
+    - retry:
+        attempts: 5
+        interval: 30
   cmd.run:
     - name: "/usr/bin/openssl pkcs8 -in /etc/pki/filebeat.key -topk8 -out /etc/pki/filebeat.p8 -nocrypt"
     - onchanges:
@@ -232,8 +238,9 @@ fbcrtlink:
       - x509: /etc/pki/registry.crt
     {%- endif %}
     - timeout: 30
-    - retry: 5
-    - interval: 30
+    - retry:
+        attempts: 5
+        interval: 30
 
 # Create a cert for the docker registry
 /etc/pki/registry.crt:
@@ -250,8 +257,9 @@ fbcrtlink:
       # Will trigger 5 days (432000 sec) from cert expiration
       - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/registry.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]'
     - timeout: 30
-    - retry: 5
-    - interval: 30
+    - retry:
+        attempts: 5
+        interval: 30
 
 regkeyperms:
   file.managed:
@@ -273,8 +281,9 @@ regkeyperms:
       - x509: /etc/pki/minio.crt
     {%- endif %}
     - timeout: 30
-    - retry: 5
-    - interval: 30
+    - retry:
+        attempts: 5
+        interval: 30
 
 # Create a cert for minio
 /etc/pki/minio.crt:
@@ -291,8 +300,9 @@ regkeyperms:
       # Will trigger 5 days (432000 sec) from cert expiration
       - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/minio.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]'
     - timeout: 30
-    - retry: 5
-    - interval: 30
+    - retry:
+        attempts: 5
+        interval: 30
 
 miniokeyperms:
   file.managed:
@@ -315,8 +325,9 @@ miniokeyperms:
       - x509: /etc/pki/elasticsearch.crt
     {%- endif %}
     - timeout: 30
-    - retry: 5
-    - interval: 30
+    - retry:
+        attempts: 5
+        interval: 30
 
 /etc/pki/elasticsearch.crt:
   x509.certificate_managed:
@@ -332,8 +343,9 @@ miniokeyperms:
       # Will trigger 5 days (432000 sec) from cert expiration
       - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/elasticsearch.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]'
     - timeout: 30
-    - retry: 5
-    - interval: 30
+    - retry:
+        attempts: 5
+        interval: 30
   cmd.run:
     - name: "/usr/bin/openssl pkcs12 -inkey /etc/pki/elasticsearch.key -in /etc/pki/elasticsearch.crt -export -out /etc/pki/elasticsearch.p12 -nodes -passout pass:"
     - onchanges:
@@ -366,8 +378,9 @@ elasticp12perms:
       - x509: /etc/pki/managerssl.crt
     {%- endif %}
     - timeout: 30
-    - retry: 5
-    - interval: 30
+    - retry:
+        attempts: 5
+        interval: 30
 
 # Create a cert for the reverse proxy
 /etc/pki/managerssl.crt:
@@ -385,8 +398,9 @@ elasticp12perms:
       # Will trigger 5 days (432000 sec) from cert expiration
       - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/managerssl.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]'
     - timeout: 30
-    - retry: 5
-    - interval: 30
+    - retry:
+        attempts: 5
+        interval: 30
 
 msslkeyperms:
   file.managed:
@@ -409,8 +423,9 @@ msslkeyperms:
       - x509: /etc/pki/fleet.crt
     {%- endif %}
     - timeout: 30
-    - retry: 5
-    - interval: 30
+    - retry:
+        attempts: 5
+        interval: 30
 
 /etc/pki/fleet.crt:
   x509.certificate_managed:
@@ -425,8 +440,9 @@ msslkeyperms:
       # Will trigger 5 days (432000 sec) from cert expiration
       - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/fleet.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]'
     - timeout: 30
-    - retry: 5
-    - interval: 30
+    - retry:
+        attempts: 5
+        interval: 30
 
 fleetkeyperms:
   file.managed:
@@ -456,8 +472,9 @@ fbcertdir:
       - x509: /opt/so/conf/filebeat/etc/pki/filebeat.crt
     {%- endif %}
     - timeout: 30
-    - retry: 5
-    - interval: 30
+    - retry:
+        attempts: 5
+        interval: 30
 
 # Request a cert and drop it where it needs to go to be distributed
 /opt/so/conf/filebeat/etc/pki/filebeat.crt:
@@ -478,8 +495,9 @@ fbcertdir:
       # Will trigger 5 days (432000 sec) from cert expiration
       - 'enddate=$(date -d "$(openssl x509 -in /opt/so/conf/filebeat/etc/pki/filebeat.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]'
     - timeout: 30
-    - retry: 5
-    - interval: 30
+    - retry:
+        attempts: 5
+        interval: 30
 
 # Convert the key to pkcs#8 so logstash will work correctly.
 filebeatpkcs:
@@ -520,8 +538,9 @@ chownfilebeatp8:
       - x509: /etc/pki/managerssl.crt
     {%- endif %}
     - timeout: 30
-    - retry: 5
-    - interval: 30
+    - retry:
+        attempts: 5
+        interval: 30
 
 # Create a cert for the reverse proxy
 /etc/pki/managerssl.crt:
@@ -539,8 +558,9 @@ chownfilebeatp8:
       # Will trigger 5 days (432000 sec) from cert expiration
       - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/managerssl.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]'
     - timeout: 30
-    - retry: 5
-    - interval: 30
+    - retry:
+        attempts: 5
+        interval: 30
 
 msslkeyperms:
   file.managed:
@@ -563,8 +583,9 @@ msslkeyperms:
       - x509: /etc/pki/fleet.crt
     {%- endif %}
     - timeout: 30
-    - retry: 5
-    - interval: 30
+    - retry:
+        attempts: 5
+        interval: 30
 
 /etc/pki/fleet.crt:
   x509.certificate_managed:
@@ -579,8 +600,9 @@ msslkeyperms:
       # Will trigger 5 days (432000 sec) from cert expiration
       - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/fleet.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]'
     - timeout: 30
-    - retry: 5
-    - interval: 30
+    - retry:
+        attempts: 5
+        interval: 30
 
 fleetkeyperms:
   file.managed:
@@ -606,8 +628,9 @@ fleetkeyperms:
       - x509: /etc/pki/elasticsearch.crt
     {%- endif %}
     - timeout: 30
-    - retry: 5
-    - interval: 30
+    - retry:
+        attempts: 5
+        interval: 30
 
 /etc/pki/elasticsearch.crt:
   x509.certificate_managed:
@@ -623,8 +646,9 @@ fleetkeyperms:
       # Will trigger 5 days (432000 sec) from cert expiration
       - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/elasticsearch.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]'
     - timeout: 30
-    - retry: 5
-    - interval: 30
+    - retry:
+        attempts: 5
+        interval: 30
   cmd.run:
     - name: "/usr/bin/openssl pkcs12 -inkey /etc/pki/elasticsearch.key -in /etc/pki/elasticsearch.crt -export -out /etc/pki/elasticsearch.p12 -nodes -passout pass:"
     - onchanges:

salt/suricata/cron/surilogcompress

@@ -1,6 +1,6 @@
 #!/bin/bash
 
 # Gzip the eve logs
-find /nsm/suricata/eve*.json -type f -printf '%T@\t%p\n' | sort -t $'\t' -g |  head -n -1 |  cut -d $'\t' -f 2 | xargs nice gzip
+find /nsm/suricata/eve*.json -type f -printf '%T@\t%p\n' | sort -t $'\t' -g |  head -n -1 |  cut -d $'\t' -f 2 | xargs nice gzip >/dev/null 2>&1
 
 # TODO Add stats log

salt/suricata/threading.map.jinja

@@ -1,4 +1,18 @@
-{% if salt['pillar.get']('sensor:suriprocs') %}
+{% if salt['pillar.get']('sensor:suripins') %}
+  {% load_yaml as cpu_affinity%}
+cpu-affinity:
+  - management-cpu-set:
+      cpu: [ {{ salt['pillar.get']('sensor:suripins')|join(",") }} ]  # include only these cpus in affinity settings
+  - receive-cpu-set:
+      cpu: [ {{ salt['pillar.get']('sensor:suripins')|join(",") }} ]  # include only these cpus in affinity settings
+  - worker-cpu-set:
+      cpu: [ {{ salt['pillar.get']('sensor:suripins')|join(",") }} ]
+      mode: "exclusive"
+      threads: {{ salt['pillar.get']('sensor:suripins')|length }}
+      prio:
+        default: "high"
+  {% endload %}
+{% elif salt['pillar.get']('sensor:suriprocs') %}
   {% load_yaml as cpu_affinity%}
 cpu-affinity:
   - management-cpu-set:
@@ -15,18 +29,4 @@ cpu-affinity:
         high: [ 3 ]
         default: "high"
   {% endload %}
-{% elif salt['pillar.get']('sensor:suripins') %}
-  {% load_yaml as cpu_affinity%}
-cpu-affinity:
-  - management-cpu-set:
-      cpu: [ {{ salt['pillar.get']('sensor:suripins')|join(",") }} ]  # include only these cpus in affinity settings
-  - receive-cpu-set:
-      cpu: [ {{ salt['pillar.get']('sensor:suripins')|join(",") }} ]  # include only these cpus in affinity settings
-  - worker-cpu-set:
-      cpu: [ {{ salt['pillar.get']('sensor:suripins')|join(",") }} ]
-      mode: "exclusive"
-      threads: {{ salt['pillar.get']('sensor:suripins')|length }}
-      prio:
-        default: "high"
-  {% endload %}
 {% endif %}

salt/systemd/reload.sls

@@ -0,0 +1,3 @@
+systemd_reload:
+  module.run:
+    - service.systemctl_reload: []

salt/telegraf/etc/telegraf.conf

@@ -17,6 +17,7 @@
 {% set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') %}
 {% set HELIX_API_KEY = salt['pillar.get']('fireeye:helix:api_key', '') %}
 {% set UNIQUEID = salt['pillar.get']('sensor:uniqueid', '') %}
+{%- set TRUE_CLUSTER = salt['pillar.get']('elasticsearch:true_cluster', False)  %}
 
 # Global tags can be specified here in key="value" format.
 [global_tags]
@@ -663,14 +664,45 @@
 
 # # Read metrics from one or more commands that can output to stdout
 
+[[inputs.exec]]
+   commands = [
+      "/scripts/sostatus.sh"
+   ]
+   data_format = "influx"
+   timeout = "15s"
+   interval = "60s"
+
+
 #   ## Commands array
-{% if grains['role'] in ['so-manager', 'so-managersearch'] %}
+{% if grains['role'] in ['so-manager'] %}
+[[inputs.exec]]
+   commands = [
+      "/scripts/redis.sh",
+      "/scripts/influxdbsize.sh",
+      "/scripts/raid.sh",
+      "/scripts/beatseps.sh"
+   ]
+  data_format = "influx"
+  ## Timeout for each command to complete.
+  timeout = "15s"
+{% elif grains['role'] in ['so-managersearch'] %}
 [[inputs.exec]]
    commands = [
       "/scripts/redis.sh",
       "/scripts/influxdbsize.sh",
       "/scripts/eps.sh",
-      "/scripts/raid.sh"
+      "/scripts/raid.sh",
+      "/scripts/beatseps.sh"
+   ]
+  data_format = "influx"
+  ## Timeout for each command to complete.
+  timeout = "15s"
+{% elif grains['role'] in ['so-node'] %}
+[[inputs.exec]]
+   commands = [
+      "/scripts/eps.sh",
+      "/scripts/raid.sh",
+      "/scripts/beatseps.sh"
    ]
   data_format = "influx"
   ## Timeout for each command to complete.
@@ -686,7 +718,8 @@
       "/scripts/zeekcaptureloss.sh",
   {% endif %}
       "/scripts/oldpcap.sh",
-      "/scripts/raid.sh"
+      "/scripts/raid.sh",
+      "/scripts/beatseps.sh"
    ]
   data_format = "influx"
   timeout = "15s"
@@ -702,7 +735,8 @@
   {% endif %}
       "/scripts/oldpcap.sh",
       "/scripts/eps.sh",
-      "/scripts/raid.sh"
+      "/scripts/raid.sh",
+      "/scripts/beatseps.sh"
    ]
   data_format = "influx"
   timeout = "15s"
@@ -720,7 +754,8 @@
   {% endif %}
       "/scripts/oldpcap.sh",
       "/scripts/eps.sh",
-      "/scripts/raid.sh"
+      "/scripts/raid.sh",
+      "/scripts/beatseps.sh"
    ]
   data_format = "influx"
   timeout = "15s"
@@ -737,7 +772,8 @@
   {% endif %}
       "/scripts/oldpcap.sh",
       "/scripts/influxdbsize.sh",
-      "/scripts/raid.sh"
+      "/scripts/raid.sh",
+      "/scripts/beatseps.sh"
    ]
   data_format = "influx"
   timeout = "15s"

salt/telegraf/init.sls

@@ -72,6 +72,8 @@ so-telegraf:
       - /opt/so/conf/telegraf/scripts:/scripts:ro
       - /opt/so/log/stenographer:/var/log/stenographer:ro
       - /opt/so/log/suricata:/var/log/suricata:ro
+      - /opt/so/log/raid:/var/log/raid:ro
+      - /opt/so/log/sostatus:/var/log/sostatus:ro
     - watch:
       - file: tgrafconf
       - file: tgrafsyncscripts

salt/telegraf/scripts/beatseps.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+#
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep)
+
+if [ ! "$THEGREP" ]; then
+
+  PREVCOUNTFILE='/tmp/beatseps.txt'
+  EVENTCOUNTCURRENT="$(curl -s localhost:5066/stats | jq '.libbeat.output.events.acked')"
+  FAILEDEVENTCOUNT="$(curl -s localhost:5066/stats | jq '.libbeat.output.events.failed')"
+
+  if [ ! -z "$EVENTCOUNTCURRENT" ]; then
+
+    if [ -f "$PREVCOUNTFILE" ]; then
+      EVENTCOUNTPREVIOUS=`cat $PREVCOUNTFILE`
+    else
+      echo "${EVENTCOUNTCURRENT}" > $PREVCOUNTFILE
+      exit 0
+    fi
+
+    echo "${EVENTCOUNTCURRENT}" > $PREVCOUNTFILE
+    # the division by 30 is because the agent interval is 30 seconds
+    EVENTS=$(((EVENTCOUNTCURRENT - EVENTCOUNTPREVIOUS)/30))
+    if [ "$EVENTS" -lt 0 ]; then
+      EVENTS=0
+    fi
+
+    echo "fbstats eps=${EVENTS%%.*},failed=$FAILEDEVENTCOUNT"
+  fi
+
+else
+    exit 0
+fi
+

salt/telegraf/scripts/checkfiles.sh

@@ -15,15 +15,13 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-APP=checkfiles
-lf=/tmp/$APP-pidLockFile
-# create empty lock file if none exists
-cat /dev/null >> $lf
-read lastPID < $lf
-# if lastPID is not null and a process with that pid exists , exit
-[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit
-echo $$ > $lf
+THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep)
 
-FILES=$(ls -1x /host/nsm/faf/complete/ | wc -l)
+if [ ! "$THEGREP" ]; then
 
-echo "faffiles files=$FILES"
+    FILES=$(ls -1x /host/nsm/strelka/unprocessed | wc -l)
+
+    echo "faffiles files=$FILES"
+else
+    exit 0
+fi

salt/telegraf/scripts/eps.sh

@@ -15,19 +15,14 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-APP=eps
-lf=/tmp/$APP-pidLockFile
-# create empty lock file if none exists
-cat /dev/null >> $lf
-read lastPID < $lf
-# if lastPID is not null and a process with that pid exists , exit
-[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit
-echo $$ > $lf
+THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep)
 
-PREVCOUNTFILE='/tmp/eps.txt'
-EVENTCOUNTCURRENT="$(curl -s localhost:9600/_node/stats | jq '.events.in')"
+if [ ! "$THEGREP" ]; then
 
-if [ ! -z "$EVENTCOUNTCURRENT" ]; then
+    PREVCOUNTFILE='/tmp/eps.txt'
+    EVENTCOUNTCURRENT="$(curl -s localhost:9600/_node/stats | jq '.events.in')"
+
+    if [ ! -z "$EVENTCOUNTCURRENT" ]; then
 
       if [ -f "$PREVCOUNTFILE" ]; then
         EVENTCOUNTPREVIOUS=`cat $PREVCOUNTFILE`
@@ -43,8 +38,9 @@ if [ ! -z "$EVENTCOUNTCURRENT" ]; then
         EVENTS=0
       fi
 
-  echo "esteps eps=${EVENTS%%.*}"
-
+      echo "consumptioneps eps=${EVENTS%%.*}"
+    fi
+else
+    exit 0
 fi
 
-exit 0

salt/telegraf/scripts/helixeps.sh

@@ -15,19 +15,14 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-APP=helixeps
-lf=/tmp/$APP-pidLockFile
-# create empty lock file if none exists
-cat /dev/null >> $lf
-read lastPID < $lf
-# if lastPID is not null and a process with that pid exists , exit
-[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit
-echo $$ > $lf
+THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep)
 
-PREVCOUNTFILE='/tmp/helixevents.txt'
-EVENTCOUNTCURRENT="$(curl -s localhost:9600/_node/stats | jq '.pipelines.helix.events.out')"
+if [ ! "$THEGREP" ]; then
 
-if [ ! -z "$EVENTCOUNTCURRENT" ]; then
+  PREVCOUNTFILE='/tmp/helixevents.txt'
+  EVENTCOUNTCURRENT="$(curl -s localhost:9600/_node/stats | jq '.pipelines.helix.events.out')"
+
+  if [ ! -z "$EVENTCOUNTCURRENT" ]; then
 
     if [ -f "$PREVCOUNTFILE" ]; then
       EVENTCOUNTPREVIOUS=`cat $PREVCOUNTFILE`
@@ -43,7 +38,7 @@ if [ ! -z "$EVENTCOUNTCURRENT" ]; then
     fi
 
     echo "helixeps eps=${EVENTS%%.*}"
-
+  fi
+else
+    exit 0
 fi
-
-exit 0

salt/telegraf/scripts/influxdbsize.sh

@@ -15,15 +15,13 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-APP=influxsize
-lf=/tmp/$APP-pidLockFile
-# create empty lock file if none exists
-cat /dev/null >> $lf
-read lastPID < $lf
-# if lastPID is not null and a process with that pid exists , exit
-[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit
-echo $$ > $lf
+THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep)
 
-INFLUXSIZE=$(du -s -k /host/nsm/influxdb | awk {'print $1'})
+if [ ! "$THEGREP" ]; then
 
-echo "influxsize kbytes=$INFLUXSIZE"
+    INFLUXSIZE=$(du -s -k /host/nsm/influxdb | awk {'print $1'})
+
+    echo "influxsize kbytes=$INFLUXSIZE"
+else
+    exit 0 
+fi

salt/telegraf/scripts/oldpcap.sh

@@ -15,18 +15,16 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-APP=oldpcap
-lf=/tmp/$APP-pidLockFile
-# create empty lock file if none exists
-cat /dev/null >> $lf
-read lastPID < $lf
-# if lastPID is not null and a process with that pid exists , exit
-[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit
-echo $$ > $lf
+THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep)
 
-# Get the data
-OLDPCAP=$(find /host/nsm/pcap -type f -exec stat -c'%n %Z' {} + | sort | grep -v "\." | head -n 1 | awk {'print $2'})
-DATE=$(date +%s)
-AGE=$(($DATE - $OLDPCAP))
+if [ ! "$THEGREP" ]; then
 
-echo "pcapage seconds=$AGE"
+    # Get the data
+    OLDPCAP=$(find /host/nsm/pcap -type f -exec stat -c'%n %Z' {} + | sort | grep -v "\." | head -n 1 | awk {'print $2'})
+    DATE=$(date +%s)
+    AGE=$(($DATE - $OLDPCAP))
+
+    echo "pcapage seconds=$AGE"
+else
+    exit 0
+fi

salt/telegraf/scripts/raid.sh

@@ -15,19 +15,17 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-APP=raid
-lf=/tmp/$APP-pidLockFile
-# create empty lock file if none exists
-cat /dev/null >> $lf
-read lastPID < $lf
-# if lastPID is not null and a process with that pid exists , exit
-[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit
-echo $$ > $lf
+THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep)
 RAIDLOG=/var/log/raid/status.log
 RAIDSTATUS=$(cat /var/log/raid/status.log)
 
-if [ -f "$RAIDLOG" ]; then
-    echo "raid raidstatus=$RAIDSTATUS "
+if [ ! "$THEGREP" ]; then
+
+    if [ -f "$RAIDLOG" ]; then
+        echo "raid $RAIDSTATUS"
+    else
+        exit 0
+    fi
 else
     exit 0
 fi

salt/telegraf/scripts/redis.sh

@@ -15,17 +15,14 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep)
 
-APP=redis
-lf=/tmp/$APP-pidLockFile
-# create empty lock file if none exists
-cat /dev/null >> $lf
-read lastPID < $lf
-# if lastPID is not null and a process with that pid exists , exit
-[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit
-echo $$ > $lf
+if [ ! "$THEGREP" ]; then
 
-UNPARSED=$(redis-cli llen logstash:unparsed | awk '{print $1}')
-PARSED=$(redis-cli llen logstash:parsed | awk '{print $1}')
+    UNPARSED=$(redis-cli llen logstash:unparsed | awk '{print $1}')
+    PARSED=$(redis-cli llen logstash:parsed | awk '{print $1}')
 
-echo "redisqueue unparsed=$UNPARSED,parsed=$PARSED"
+    echo "redisqueue unparsed=$UNPARSED,parsed=$PARSED"
+else
+   exit 0
+fi

salt/telegraf/scripts/sostatus.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+#
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep)
+
+if [ ! "$THEGREP" ]; then
+
+    SOSTATUSLOG=/var/log/sostatus/status.log
+    SOSTATUSSTATUS=$(cat /var/log/sostatus/status.log)
+
+    if [ -f "$SOSTATUSLOG" ]; then
+        echo "sostatus status=$SOSTATUSSTATUS"
+    else
+        exit 0
+    fi
+else 
+    exit 0
+fi

salt/telegraf/scripts/stenoloss.sh

@@ -15,31 +15,29 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-APP=stenoloss
-lf=/tmp/$APP-pidLockFile
-# create empty lock file if none exists
-cat /dev/null >> $lf
-read lastPID < $lf
-# if lastPID is not null and a process with that pid exists , exit
-[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit
-echo $$ > $lf
+THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep)
 
-TSFILE=/var/log/telegraf/laststenodrop.log
-if [ -f "$TSFILE" ]; then
+if [ ! "$THEGREP" ]; then
+
+    TSFILE=/var/log/telegraf/laststenodrop.log
+    if [ -f "$TSFILE" ]; then
         LASTTS=$(cat $TSFILE)
-else
+    else
         LASTTS=0
-fi
+    fi
 
-# Get the data
-LOGLINE=$(tac /var/log/stenographer/stenographer.log | grep -m1 drop)
-CURRENTTS=$(echo $LOGLINE | awk '{print $1}')
+    # Get the data
+    LOGLINE=$(tac /var/log/stenographer/stenographer.log | grep -m1 drop)
+    CURRENTTS=$(echo $LOGLINE | awk '{print $1}')
 
-if [[ "$CURRENTTS" != "$LASTTS" ]]; then
+    if [[ "$CURRENTTS" != "$LASTTS" ]]; then
       DROP=$(echo $LOGLINE | awk '{print $14}' | awk -F "=" '{print $2}')
       echo $CURRENTTS > $TSFILE
-else
+    else
       DROP=0
-fi
+    fi
 
-echo "stenodrop drop=$DROP"
+    echo "stenodrop drop=$DROP"
+else
+    exit 0
+fi

salt/telegraf/scripts/suriloss.sh

@@ -16,19 +16,14 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 
-APP=suriloss
-lf=/tmp/$APP-pidLockFile
-# create empty lock file if none exists
-cat /dev/null >> $lf
-read lastPID < $lf
-# if lastPID is not null and a process with that pid exists , exit
-[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit
-echo $$ > $lf
+THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep)
 
-SURILOG=$(tac /var/log/suricata/stats.log | grep kernel | head -4)
-CHECKIT=$(echo $SURILOG | grep -o 'drop' | wc -l)
+if [ ! "$THEGREP" ]; then
 
-if [ $CHECKIT == 2 ]; then
+    SURILOG=$(tac /var/log/suricata/stats.log | grep kernel | head -4)
+    CHECKIT=$(echo $SURILOG | grep -o 'drop' | wc -l)
+
+    if [ $CHECKIT == 2 ]; then
       declare RESULT=($SURILOG)
 
       CURRENTDROP=${RESULT[4]}
@@ -47,6 +42,7 @@ if [ $CHECKIT == 2 ]; then
         LOSS=$(echo 4 k $DROPPED $TOTAL / p | dc)
         echo "suridrop drop=$LOSS"
       fi
+    fi
 else
   echo "suridrop drop=0"
 fi

salt/telegraf/scripts/zeekcaptureloss.sh

@@ -18,28 +18,23 @@
 
 # This script returns the average of all the workers average capture loss to telegraf / influxdb in influx format include nanosecond precision timestamp
 
-APP=zeekcaploss
-lf=/tmp/$APP-pidLockFile
-# create empty lock file if none exists
-cat /dev/null >> $lf
-read lastPID < $lf
-# if lastPID is not null and a process with that pid exists , exit
-[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit
-echo $$ > $lf
+THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep)
 
-if [ -d "/host/nsm/zeek/spool/logger" ]; then
+if [ ! "$THEGREP" ]; then
+
+    if [ -d "/host/nsm/zeek/spool/logger" ]; then
       WORKERS={{ salt['pillar.get']('sensor:zeek_lbprocs', salt['pillar.get']('sensor:zeek_pins') | length) }}
       ZEEKLOG=/host/nsm/zeek/spool/logger/capture_loss.log
-elif [ -d "/host/nsm/zeek/spool/zeeksa" ]; then
+    elif [ -d "/host/nsm/zeek/spool/zeeksa" ]; then
       WORKERS=1
       ZEEKLOG=/host/nsm/zeek/spool/zeeksa/capture_loss.log
-else
+    else
       echo 'Zeek capture_loss.log not found' >/dev/stderr
       exit 2
-fi
+    fi
 
-LASTCAPTURELOSSLOG=/var/log/telegraf/lastcaptureloss.txt
-if [ -f "$ZEEKLOG" ]; then
+    LASTCAPTURELOSSLOG=/var/log/telegraf/lastcaptureloss.txt
+    if [ -f "$ZEEKLOG" ]; then
       CURRENTTS=$(tail -1 $ZEEKLOG | jq .ts | sed 's/"//g')
       if [ -f "$LASTCAPTURELOSSLOG" ]; then
         LASTTS=$(cat $LASTCAPTURELOSSLOG)
@@ -49,4 +44,7 @@ if [ -f "$ZEEKLOG" ]; then
         fi
       fi
       echo "$CURRENTTS" > $LASTCAPTURELOSSLOG
+    fi
+else
+    exit 0
 fi

salt/telegraf/scripts/zeekloss.sh

@@ -17,34 +17,32 @@
 
 # This script returns the packets dropped by Zeek, but it isn't a percentage. $LOSS * 100 would be the percentage
 
-APP=zeekloss
-lf=/tmp/$APP-pidLockFile
-# create empty lock file if none exists
-cat /dev/null >> $lf
-read lastPID < $lf
-# if lastPID is not null and a process with that pid exists , exit
-[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit
-echo $$ > $lf
+THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep)
 
-ZEEKLOG=$(tac /host/nsm/zeek/logs/packetloss.log | head -2)
-declare RESULT=($ZEEKLOG)
-CURRENTDROP=${RESULT[3]}
-# zeek likely not running if this is true
-if [[ $CURRENTDROP == "rcvd:" ]]; then
+if [ ! "$THEGREP" ]; then
+
+  ZEEKLOG=$(tac /host/nsm/zeek/logs/packetloss.log | head -2)
+  declare RESULT=($ZEEKLOG)
+  CURRENTDROP=${RESULT[3]}
+  # zeek likely not running if this is true
+  if [[ $CURRENTDROP == "rcvd:" ]]; then
     CURRENTDROP=0
     PASTDROP=0
     DROPPED=0
-else
+  else
     PASTDROP=${RESULT[9]}
     DROPPED=$((CURRENTDROP - PASTDROP))
-fi
-if [[ "$DROPPED" -le 0 ]]; then
+  fi
+  if [[ "$DROPPED" -le 0 ]]; then
     LOSS=0
     echo "zeekdrop drop=0"
-else
+  else
     CURRENTPACKETS=${RESULT[5]}
     PASTPACKETS=${RESULT[11]}
     TOTAL=$((CURRENTPACKETS - PASTPACKETS))
     LOSS=$(echo 4 k $DROPPED $TOTAL / p | dc)
     echo "zeekdrop drop=$LOSS"
+  fi
+else
+  exit 0
 fi

salt/top.sls

@@ -14,27 +14,21 @@
 {% set CURATOR = salt['pillar.get']('curator:enabled', True) %}
 {% set REDIS = salt['pillar.get']('redis:enabled', True) %}
 {% set STRELKA = salt['pillar.get']('strelka:enabled', '0') %}
-{% set ISAIRGAP = salt['pillar.get']('global:airgap', 'False') %}
 {% import_yaml 'salt/minion.defaults.yaml' as saltversion %}
 {% set saltversion = saltversion.salt.minion.version %}
+{% set INSTALLEDSALTVERSION = grains.saltversion %}
 
 base:
 
   'not G@saltversion:{{saltversion}}':
     - match: compound
     - salt.minion-state-apply-test
-    {% if ISAIRGAP is sameas true %}
-    - airgap
-    {% endif %}
+    - repo.client
     - salt.minion
 
   'G@os:CentOS and G@saltversion:{{saltversion}}':
     - match: compound
-    {% if ISAIRGAP is sameas true %}
-    - airgap
-    {% else %}
-    - yum
-    {% endif %}
+    - repo.client
     - yum.packages
 
   '* and G@saltversion:{{saltversion}}':

salt/yum/init.sls

@@ -1,17 +0,0 @@
-{% from 'allowed_states.map.jinja' import allowed_states %}
-{% if sls in allowed_states %}
-
-yumconf:
-  file.managed:
-    - name: /etc/yum.conf
-    - source: salt://yum/etc/yum.conf.jinja
-    - mode: 644
-    - template: jinja
-
-{% else %}
-
-{{sls}}_state_not_allowed:
-  test.fail_without_changes:
-    - name: {{sls}}_state_not_allowed
-
-{% endif %}

salt/zeek/init.sls

@@ -78,6 +78,7 @@ zeekspoolownership:
   file.directory:
     - name: /nsm/zeek/spool
     - user: 937
+    - max_depth: 0
     - recurse:
       - user
 

setup/automation/distributed-airgap-manager

@@ -42,7 +42,6 @@ INTERWEBS=AIRGAP
 # LSPIPELINEBATCH=
 # LSPIPELINEWORKERS=
 MANAGERADV=BASIC
-MANAGERUPDATES=1
 # MDNS=
 # MGATEWAY=
 # MIP=

setup/automation/distributed-airgap-search

@@ -35,6 +35,7 @@ ADMINPASS2=onionuser
 HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12
 HNSENSOR=inherit
 HOSTNAME=distributed-search
+INTERWEBS=AIRGAP
 install_type=SEARCHNODE
 # LSINPUTBATCHCOUNT=
 # LSINPUTTHREADS=