Merge pull request #4254 from Security-Onion-Solutions/2.3.51

2.3.51

.github/workflows/leaktest.yml

@@ -1,6 +1,6 @@
 name: leak-test
 
-on: [push,pull_request]
+on: [pull_request]
 
 jobs:
   build:

README.md

@@ -1,6 +1,6 @@
-## Security Onion 2.3.20
+## Security Onion 2.3.51
 
-Security Onion 2.3.20 is here!
+Security Onion 2.3.51 is here!
 
 ## Screenshots
 

VERIFY_ISO.md

@@ -1,16 +1,17 @@
-### 2.3.20 ISO image built on 2020/12/20
+### 2.3.51 ISO image built on 2021/04/27
+
 
 ### Download and Verify
 
-2.3.20 ISO image:  
-https://download.securityonion.net/file/securityonion/securityonion-2.3.20.iso
+2.3.51 ISO image:  
+https://download.securityonion.net/file/securityonion/securityonion-2.3.51.iso
 
-MD5: E348FA65A46FD3FBA0D574D9C1A0582D  
-SHA1: 4A6E6D4E0B31ECA1B72E642E3DB2C186B59009D6  
-SHA256: 25DE77097903640771533FA13094D0720A032B70223875F8C77A92F5C44CA687 
+MD5: 7CFB525BEFC0A9F2ED148F5831E387FA  
+SHA1: 8CC34FCCC36822B309B8168AA706B3D1EC7F3BFD  
+SHA256: 9892C2546C9AE5A48015160F379B070F0BE30C89693B97F3F1E1592DDCE1DEE0 
 
 Signature for ISO image:  
-https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.20.iso.sig
+https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.51.iso.sig
 
 Signing key:  
 https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS  
@@ -24,22 +25,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/ma
 
 Download the signature file for the ISO:  
 ```
-wget https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.20.iso.sig
+wget https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.51.iso.sig
 ```
 
 Download the ISO image:  
 ```
-wget https://download.securityonion.net/file/securityonion/securityonion-2.3.20.iso
+wget https://download.securityonion.net/file/securityonion/securityonion-2.3.51.iso
 ```
 
 Verify the downloaded ISO image using the signature file:  
 ```
-gpg --verify securityonion-2.3.20.iso.sig securityonion-2.3.20.iso
+gpg --verify securityonion-2.3.51.iso.sig securityonion-2.3.51.iso
 ```
 
 The output should show "Good signature" and the Primary key fingerprint should match what's shown below:
 ```
-gpg: Signature made Sun 20 Dec 2020 11:11:28 AM EST using RSA key ID FE507013
+gpg: Signature made Thu 20 May 2021 07:49:57 AM EDT using RSA key ID FE507013
 gpg: Good signature from "Security Onion Solutions, LLC <info@securityonionsolutions.com>"
 gpg: WARNING: This key is not certified with a trusted signature!
 gpg:          There is no indication that the signature belongs to the owner.

VERSION

@@ -1 +1 @@
-2.3.20
+2.3.51

files/salt/master/master

@@ -13,6 +13,8 @@
 # user: socore
 
 log_file: /opt/so/log/salt/master
+log_level_logfile: info
+log_level: info
 
 #####      File Server settings      #####
 ##########################################

pillar/docker/config.sls

@@ -1,208 +0,0 @@
-{%- set FLEETMANAGER = salt['pillar.get']('global:fleet_manager', False) -%}
-{%- set FLEETNODE = salt['pillar.get']('global:fleet_node', False) -%}
-{% set WAZUH = salt['pillar.get']('manager:wazuh', '0') %}
-{% set THEHIVE = salt['pillar.get']('manager:thehive', '0') %}
-{% set PLAYBOOK = salt['pillar.get']('manager:playbook', '0') %}
-{% set FREQSERVER = salt['pillar.get']('manager:freq', '0') %}
-{% set DOMAINSTATS = salt['pillar.get']('manager:domainstats', '0') %}
-{% set ZEEKVER = salt['pillar.get']('global:mdengine', 'COMMUNITY') %}
-{% set GRAFANA = salt['pillar.get']('manager:grafana', '0') %}
-
-eval:
-  containers:
-    - so-nginx
-    - so-telegraf
-    {% if  GRAFANA == '1' %}
-    - so-influxdb
-    - so-grafana
-    {% endif %}
-    - so-dockerregistry
-    - so-soc
-    - so-kratos
-    - so-idstools
-    {% if FLEETMANAGER %}
-    - so-mysql
-    - so-fleet
-    - so-redis
-    {% endif %}
-    - so-elasticsearch
-    - so-logstash
-    - so-kibana
-    - so-steno
-    - so-suricata
-    - so-zeek
-    - so-curator
-    - so-elastalert
-    {% if WAZUH != '0' %}
-    - so-wazuh
-    {% endif %}
-    - so-soctopus
-    {% if THEHIVE != '0' %}
-    - so-thehive
-    - so-thehive-es
-    - so-cortex
-    {% endif %}
-    {% if PLAYBOOK != '0' %}
-    - so-playbook
-    {% endif %}
-    {% if FREQSERVER != '0' %}
-    - so-freqserver
-    {% endif %}
-    {% if DOMAINSTATS != '0' %}
-    - so-domainstats
-    {% endif %}
-heavy_node:
-  containers:
-    - so-nginx
-    - so-telegraf
-    - so-redis
-    - so-logstash
-    - so-elasticsearch
-    - so-curator
-    - so-steno
-    - so-suricata
-    - so-wazuh
-    - so-filebeat
-    {% if ZEEKVER != 'SURICATA' %}
-    - so-zeek
-    {% endif %}
-helix:
-  containers:
-    - so-nginx
-    - so-telegraf
-    - so-idstools
-    - so-steno
-    - so-zeek
-    - so-redis
-    - so-logstash
-    - so-filebeat
-hot_node:
-  containers:
-    - so-nginx
-    - so-telegraf
-    - so-logstash
-    - so-elasticsearch
-    - so-curator
-manager_search:
-  containers:
-    - so-nginx
-    - so-telegraf
-    - so-soc
-    - so-kratos
-    - so-acng
-    - so-idstools
-    - so-redis
-    - so-logstash
-    - so-elasticsearch
-    - so-curator
-    - so-kibana
-    - so-elastalert
-    - so-filebeat
-    - so-soctopus
-    {% if FLEETMANAGER %}
-    - so-mysql
-    - so-fleet
-    - so-redis
-    {% endif %}
-    {% if WAZUH != '0' %}
-    - so-wazuh
-    {% endif %}
-    - so-soctopus
-    {% if THEHIVE != '0' %}
-    - so-thehive
-    - so-thehive-es
-    - so-cortex
-    {% endif %}
-    {% if PLAYBOOK != '0' %}
-    - so-playbook
-    {% endif %}
-    {% if FREQSERVER != '0' %}
-    - so-freqserver
-    {% endif %}
-    {% if DOMAINSTATS != '0' %}
-    - so-domainstats
-    {% endif %}
-manager:
-  containers:
-    - so-dockerregistry
-    - so-nginx
-    - so-telegraf
-    {% if  GRAFANA == '1' %}
-    - so-influxdb
-    - so-grafana
-    {% endif %}
-    - so-soc
-    - so-kratos
-    - so-acng
-    - so-idstools
-    - so-redis
-    - so-elasticsearch
-    - so-logstash
-    - so-kibana
-    - so-elastalert
-    - so-filebeat
-    {% if FLEETMANAGER %}
-    - so-mysql
-    - so-fleet
-    - so-redis
-    {% endif %}
-    {% if WAZUH != '0' %}
-    - so-wazuh
-    {% endif %}
-    - so-soctopus
-    {% if THEHIVE != '0' %}
-    - so-thehive
-    - so-thehive-es
-    - so-cortex
-    {% endif %}
-    {% if PLAYBOOK != '0' %}
-    - so-playbook
-    {% endif %}
-    {% if FREQSERVER != '0' %}
-    - so-freqserver
-    {% endif %}
-    {% if DOMAINSTATS != '0' %}
-    - so-domainstats
-    {% endif %}
-parser_node:
-  containers:
-    - so-nginx
-    - so-telegraf
-    - so-logstash
-search_node:
-  containers:
-    - so-nginx
-    - so-telegraf
-    - so-logstash
-    - so-elasticsearch
-    - so-curator
-    - so-filebeat
-    {% if WAZUH != '0' %}
-    - so-wazuh
-    {% endif %}
-sensor:
-  containers:
-    - so-nginx
-    - so-telegraf
-    - so-steno
-    - so-suricata
-    {% if ZEEKVER != 'SURICATA' %}
-    - so-zeek
-    {% endif %}
-    - so-wazuh
-    - so-filebeat
-warm_node:
-  containers:
-    - so-nginx
-    - so-telegraf
-    - so-elasticsearch
-fleet:
-  containers:
-    {% if FLEETNODE %}
-    - so-mysql
-    - so-fleet
-    - so-redis
-    - so-filebeat
-    - so-nginx
-    - so-telegraf
-    {% endif %}

pillar/logrotate/init.sls

@@ -9,3 +9,5 @@ logrotate:
     extension .log
     dateext
     dateyesterday
+  group_conf: |
+    su root socore

salt/airgap/init.sls

@@ -1,60 +0,0 @@
-{% set MANAGER = salt['grains.get']('master') %}
-airgapyum:
-  file.managed:
-    - name: /etc/yum/yum.conf
-    - source: salt://airgap/files/yum.conf
-
-airgap_repo:
-  pkgrepo.managed:
-    - humanname: Airgap Repo
-    - baseurl: https://{{ MANAGER }}/repo
-    - gpgcheck: 0
-    - sslverify: 0
-
-agbase:
-  file.absent:
-    - name: /etc/yum.repos.d/CentOS-Base.repo
-
-agcr:
-  file.absent:
-    - name: /etc/yum.repos.d/CentOS-CR.repo
-
-agdebug:
-  file.absent:
-    - name: /etc/yum.repos.d/CentOS-Debuginfo.repo
-
-agfasttrack:
-  file.absent:
-    - name: /etc/yum.repos.d/CentOS-fasttrack.repo
-
-agmedia:
-  file.absent:
-    - name: /etc/yum.repos.d/CentOS-Media.repo
-
-agsources:
-  file.absent:
-    - name: /etc/yum.repos.d/CentOS-Sources.repo
-
-agvault:
-  file.absent:
-    - name: /etc/yum.repos.d/CentOS-Vault.repo
-
-agkernel:
-  file.absent:
-    - name: /etc/yum.repos.d/CentOS-x86_64-kernel.repo
-
-agepel:
-  file.absent:
-    - name: /etc/yum.repos.d/epel.repo
-
-agtesting:
-  file.absent:
-    - name: /etc/yum.repos.d/epel-testing.repo
-
-agssrepo:
-  file.absent:
-    - name: /etc/yum.repos.d/saltstack.repo
-
-agwazrepo:
-  file.absent:
-    - name: /etc/yum.repos.d/wazuh.repo

salt/allowed_states.map.jinja

@@ -0,0 +1,299 @@
+{% set ZEEKVER = salt['pillar.get']('global:mdengine', '') %}
+{% set WAZUH = salt['pillar.get']('global:wazuh', '0') %}
+{% set THEHIVE = salt['pillar.get']('manager:thehive', '0') %}
+{% set PLAYBOOK = salt['pillar.get']('manager:playbook', '0') %}
+{% set FREQSERVER = salt['pillar.get']('manager:freq', '0') %}
+{% set DOMAINSTATS = salt['pillar.get']('manager:domainstats', '0') %}
+{% set FLEETMANAGER = salt['pillar.get']('global:fleet_manager', False) %}
+{% set FLEETNODE = salt['pillar.get']('global:fleet_node', False) %}
+{% set ELASTALERT = salt['pillar.get']('elastalert:enabled', True) %}
+{% set ELASTICSEARCH = salt['pillar.get']('elasticsearch:enabled', True) %}
+{% set FILEBEAT = salt['pillar.get']('filebeat:enabled', True) %}
+{% set KIBANA = salt['pillar.get']('kibana:enabled', True) %}
+{% set LOGSTASH = salt['pillar.get']('logstash:enabled', True) %}
+{% set CURATOR = salt['pillar.get']('curator:enabled', True) %}
+{% set REDIS = salt['pillar.get']('redis:enabled', True) %}
+{% set STRELKA = salt['pillar.get']('strelka:enabled', '0') %}
+{% set ISAIRGAP = salt['pillar.get']('global:airgap', False) %}
+{% import_yaml 'salt/minion.defaults.yaml' as saltversion %}
+{% set saltversion = saltversion.salt.minion.version %}
+
+{# this is the list we are returning from this map file, it gets built below #}
+{% set allowed_states= [] %}
+
+{% if grains.saltversion | string == saltversion | string %}
+
+  {% set allowed_states= salt['grains.filter_by']({
+      'so-eval': [
+          'salt.master',
+          'ca',
+          'ssl',
+          'registry',
+          'manager',
+          'nginx',
+          'telegraf',
+          'influxdb',
+          'grafana',
+          'soc',
+          'firewall',
+          'idstools',
+          'suricata.manager',
+          'healthcheck',
+          'pcap',
+          'suricata',
+          'utility',
+          'schedule',
+          'soctopus',
+          'tcpreplay',
+          'docker_clean'
+          ],
+      'so-heavynode': [
+          'ca',
+          'ssl',
+          'nginx',
+          'telegraf',
+          'firewall',
+          'pcap',
+          'suricata',
+          'healthcheck',
+          'schedule',
+          'tcpreplay',
+          'docker_clean'
+          ],
+      'so-helixsensor': [
+          'salt.master',
+          'ca',
+          'ssl',
+          'registry',
+          'telegraf',
+          'firewall',
+          'idstools',
+          'suricata.manager',
+          'zeek',
+          'redis',
+          'elasticsearch',
+          'logstash',
+          'schedule',
+          'tcpreplay',
+          'docker_clean'
+          ],
+      'so-fleet': [
+          'ca',
+          'ssl',
+          'nginx',
+          'telegraf',
+          'firewall',
+          'mysql',
+          'redis',
+          'fleet',
+          'fleet.install_package',
+          'filebeat',
+          'schedule',
+          'docker_clean'
+          ],
+      'so-import': [
+          'salt.master',
+          'ca',
+          'ssl',
+          'registry',
+          'manager',
+          'nginx',
+          'soc',
+          'firewall',
+          'idstools',
+          'suricata.manager',
+          'pcap',
+          'utility',
+          'suricata',
+          'zeek',
+          'schedule',
+          'tcpreplay',
+          'docker_clean'
+          ],
+      'so-manager': [
+          'salt.master',
+          'ca',
+          'ssl',
+          'registry',
+          'manager',
+          'nginx',
+          'telegraf',
+          'influxdb',
+          'grafana',
+          'soc',
+          'firewall',
+          'idstools',
+          'suricata.manager',
+          'utility',
+          'schedule',
+          'soctopus',
+          'docker_clean'
+          ],
+      'so-managersearch': [
+          'salt.master',
+          'ca',
+          'ssl',
+          'registry',
+          'nginx',
+          'telegraf',
+          'influxdb',
+          'grafana',
+          'soc',
+          'firewall',
+          'manager',
+          'idstools',
+          'suricata.manager',
+          'utility',
+          'schedule',
+          'soctopus',
+          'docker_clean'
+          ],
+      'so-node': [
+          'ca',
+          'ssl',
+          'nginx',
+          'telegraf',
+          'firewall',
+          'schedule',
+          'docker_clean'
+          ],
+      'so-standalone': [
+          'salt.master',
+          'ca',
+          'ssl',
+          'registry',
+          'manager',
+          'nginx',
+          'telegraf',
+          'influxdb',
+          'grafana',
+          'soc',
+          'firewall',
+          'idstools',
+          'suricata.manager',
+          'pcap',
+          'suricata',
+          'healthcheck',
+          'utility',
+          'schedule',
+          'soctopus',
+          'tcpreplay',
+          'docker_clean'
+          ],
+      'so-sensor': [
+          'ca',
+          'ssl',
+          'telegraf',
+          'firewall',
+          'nginx',
+          'pcap',
+          'suricata',
+          'healthcheck',
+          'wazuh',
+          'filebeat',
+          'schedule',
+          'tcpreplay',
+          'docker_clean'
+          ],
+  }, grain='role') %}
+
+  {% if FILEBEAT and grains.role in ['so-helixsensor', 'so-eval', 'so-manager', 'so-standalone', 'so-node', 'so-managersearch', 'so-heavynode', 'so-import'] %}
+    {% do allowed_states.append('filebeat') %}
+  {% endif %}
+
+  {% if ((FLEETMANAGER or FLEETNODE) or PLAYBOOK != 0) and grains.role in ['so-eval', 'so-manager', 'so-managersearch', 'so-standalone'] %}
+    {% do allowed_states.append('mysql') %}
+  {% endif %}
+
+  {% if (FLEETMANAGER or FLEETNODE) and grains.role in ['so-sensor', 'so-eval', 'so-manager', 'so-standalone', 'so-node', 'so-managersearch', 'so-heavynode'] %}
+    {% do allowed_states.append('fleet.install_package') %}
+  {% endif %}
+
+  {% if (FLEETMANAGER or FLEETNODE) and grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-managersearch', 'so-heavynode'] %}
+    {% do allowed_states.append('fleet') %}
+  {% endif %}
+
+  {% if (FLEETMANAGER or FLEETNODE) and grains.role in ['so-eval'] %}
+    {% do allowed_states.append('redis') %}
+  {% endif %}
+
+  {%- if ZEEKVER != 'SURICATA' and grains.role in ['so-sensor', 'so-eval', 'so-standalone', 'so-heavynode'] %}
+    {% do allowed_states.append('zeek') %}
+  {%- endif %}
+
+  {% if STRELKA and grains.role in ['so-sensor', 'so-eval', 'so-standalone', 'so-heavynode'] %}
+    {% do allowed_states.append('strelka') %}
+  {% endif %}
+
+  {% if WAZUH and grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-node', 'so-managersearch', 'so-heavynode']%}
+    {% do allowed_states.append('wazuh') %}
+  {% endif %}
+
+  {% if ELASTICSEARCH and grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-node', 'so-managersearch', 'so-heavynode', 'so-import'] %}
+    {% do allowed_states.append('elasticsearch') %}
+  {% endif %}
+
+  {% if KIBANA and grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-managersearch', 'so-import'] %}
+    {% do allowed_states.append('kibana') %}
+  {% endif %}
+
+  {% if CURATOR and grains.role in ['so-eval', 'so-standalone', 'so-node', 'so-managersearch', 'so-heavynode'] %}
+    {% do allowed_states.append('curator') %}
+  {% endif %}
+
+  {% if ELASTALERT and grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-managersearch'] %}
+    {% do allowed_states.append('elastalert') %}
+  {% endif %}
+
+  {% if (THEHIVE != 0) and grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-managersearch'] %}
+    {% do allowed_states.append('thehive') %}
+  {% endif %}
+
+  {% if (PLAYBOOK !=0) and grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-managersearch'] %}
+    {% do allowed_states.append('playbook') %}
+  {% endif %}
+
+  {% if (PLAYBOOK !=0) and grains.role in ['so-eval'] %}
+    {% do allowed_states.append('redis') %}
+  {% endif %}
+
+  {% if (FREQSERVER !=0) and grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-managersearch'] %}
+    {% do allowed_states.append('freqserver') %}
+  {% endif %}
+
+  {% if (DOMAINSTATS !=0) and grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-managersearch'] %}
+    {% do allowed_states.append('domainstats') %}
+  {% endif %}
+
+  {% if LOGSTASH and grains.role in ['so-helixsensor', 'so-manager', 'so-standalone', 'so-node', 'so-managersearch', 'so-heavynode'] %}
+    {% do allowed_states.append('logstash') %}
+  {% endif %}
+
+  {% if REDIS and grains.role in ['so-manager', 'so-standalone', 'so-managersearch', 'so-heavynode'] %}
+    {% do allowed_states.append('redis') %}
+  {% endif %}
+
+  {% if grains.os == 'CentOS' %}
+    {% if not ISAIRGAP %}
+      {% do allowed_states.append('yum') %}
+    {% endif %}
+    {% do allowed_states.append('yum.packages') %}
+  {% endif %}
+
+  {# all nodes on the right salt version can run the following states #}
+  {% do allowed_states.append('common') %}
+  {% do allowed_states.append('patch.os.schedule') %}
+  {% do allowed_states.append('motd') %}
+  {% do allowed_states.append('salt.minion-check') %}
+  {% do allowed_states.append('sensoroni') %}
+  {% do allowed_states.append('salt.lasthighstate') %}
+
+{% endif %}
+
+
+{% if ISAIRGAP %}
+  {% do allowed_states.append('airgap') %}
+{% endif %}
+
+{# all nodes can always run salt.minion state #}
+{% do allowed_states.append('salt.minion') %}

salt/ca/init.sls

@@ -1,7 +1,5 @@
-{% set show_top = salt['state.show_top']() %}
-{% set top_states = show_top.values() | join(', ') %}
-
-{% if 'ca' in top_states %}
+{% from 'allowed_states.map.jinja' import allowed_states %}
+{% if sls in allowed_states %}
 
 {% set manager = salt['grains.get']('master') %}
 /etc/salt/minion.d/signing_policies.conf:
@@ -44,6 +42,10 @@ pki_private_key:
     - replace: False
     - require:
       - file: /etc/pki
+    - timeout: 30
+    - retry:
+        attempts: 5
+        interval: 30
 
 x509_pem_entries:
   module.run:
@@ -60,8 +62,8 @@ cakeyperms:
 
 {% else %}
 
-ca_state_not_allowed:
+{{sls}}_state_not_allowed:
   test.fail_without_changes:
-    - name: ca_state_not_allowed
+    - name: {{sls}}_state_not_allowed
 
 {% endif %}

salt/common/cron/common-rotate

@@ -1,2 +1,2 @@
 #!/bin/bash
-logrotate -f /opt/so/conf/log-rotate.conf >/dev/null 2>&1
+/usr/sbin/logrotate -f /opt/so/conf/log-rotate.conf > /dev/null 2>&1

salt/common/files/99-reserved-ports.conf

@@ -0,0 +1 @@
+net.ipv4.ip_local_reserved_ports=55000,57314,47760-47860

salt/common/files/log-rotate.conf

@@ -1,4 +1,6 @@
 {%- set logrotate_conf = salt['pillar.get']('logrotate:conf') %}
+{%- set group_conf = salt['pillar.get']('logrotate:group_conf') %}
+
 
 /opt/so/log/aptcacher-ng/*.log
 /opt/so/log/idstools/*.log
@@ -13,12 +15,21 @@
 /opt/so/log/fleet/*.log
 /opt/so/log/suricata/*.log
 /opt/so/log/mysql/*.log
-/opt/so/log/playbook/*.log
-/opt/so/log/logstash/*.log
-/opt/so/log/filebeat/*.log
 /opt/so/log/telegraf/*.log
 /opt/so/log/redis/*.log
+/opt/so/log/sensoroni/*.log
+/opt/so/log/stenographer/*.log
 /opt/so/log/salt/so-salt-minion-check
+/opt/so/log/salt/minion
+/opt/so/log/salt/master
 {
     {{ logrotate_conf | indent(width=4) }}
 }
+
+# Playbook's log directory needs additional configuration
+# because Playbook requires a more permissive directory
+/opt/so/log/playbook/*.log
+{
+    {{ logrotate_conf | indent(width=4) }}
+    {{ group_conf | indent(width=4) }}
+}

salt/common/files/sensor-rotate.conf

@@ -6,5 +6,17 @@
     nocompress
     create
     sharedscripts
-    endscript
+}
+
+/nsm/strelka/log/strelka.log
+{
+    daily
+    rotate 14
+    missingok
+    copytruncate
+    compress
+    create
+    extension .log
+    dateext
+    dateyesterday
 }

salt/common/files/soversion

@@ -0,0 +1,2 @@
+{%- set VERSION = salt['pillar.get']('global:soversion') -%}
+{{ VERSION }}

salt/common/files/vimrc

@@ -0,0 +1,6 @@
+" Activates filetype detection
+filetype plugin indent on
+
+" Sets .sls files to use YAML syntax highlighting
+autocmd BufNewFile,BufRead *.sls set syntax=yaml
+set number

salt/common/init.sls

@@ -1,7 +1,5 @@
-{% set show_top = salt['state.show_top']() %}
-{% set top_states = show_top.values() | join(', ') %}
-
-{% if 'common' in top_states %}
+{% from 'allowed_states.map.jinja' import allowed_states %}
+{% if sls in allowed_states %}
 
 {% set role = grains.id.split('_') | last %}
 
@@ -51,6 +49,11 @@ sosaltstackperms:
     - gid: 939
     - dir_mode: 770
 
+so_log_perms:
+  file.directory:
+    - name: /opt/so/log
+    - dir_mode: 755
+
 # Create a state directory
 statedir:
   file.directory:
@@ -66,20 +69,12 @@ salttmp:
     - group: 939
     - makedirs: True
 
-# Install epel
-{% if grains['os'] == 'CentOS' %}
-repair_yumdb:
-  cmd.run:
-    - name: 'mv -f /var/lib/rpm/__db* /tmp && yum clean all'
-    - onlyif:
-      - 'yum check-update 2>&1 | grep "Error: rpmdb open failed"'
-
-epel:
-  pkg.installed:
-    - skip_suggestions: True
-    - pkgs:
-      - epel-release
-{% endif %}
+# VIM config
+vimconfig:
+  file.managed:
+    - name: /root/.vimrc
+    - source: salt://common/files/vimrc
+    - replace: False
 
 # Install common packages
 {% if grains['os'] != 'CentOS' %}     
@@ -92,7 +87,6 @@ commonpkgs:
       - ntpdate
       - jq
       - python3-docker
-      - docker-ce
       - curl
       - ca-certificates
       - software-properties-common
@@ -106,12 +100,17 @@ commonpkgs:
       - python3-dateutil
       - python3-m2crypto
       - python3-mysqldb
+      - python3-packaging
       - git
+      - vim
+
 heldpackages:
   pkg.installed:
     - pkgs:
-      - containerd.io: 1.2.13-2
-      - docker-ce: 5:19.03.14~3-0~ubuntu-bionic
+      - containerd.io: 1.4.4-1
+      - docker-ce: 5:20.10.5~3-0~ubuntu-bionic
+      - docker-ce-cli: 5:20.10.5~3-0~ubuntu-bionic
+      - docker-ce-rootless-extras: 5:20.10.5~3-0~ubuntu-bionic
     - hold: True
     - update_holds: True
 
@@ -137,17 +136,21 @@ commonpkgs:
       - python36-dateutil
       - python36-m2crypto
       - python36-mysql
+      - python36-packaging
       - yum-utils
       - device-mapper-persistent-data
       - lvm2
       - openssl
       - git
+      - vim-enhanced
 
 heldpackages:
   pkg.installed:
     - pkgs:
-      - containerd.io: 1.2.13-3.2.el7
-      - docker-ce: 3:19.03.14-3.el7
+      - containerd.io: 1.4.4-3.1.el7
+      - docker-ce: 3:20.10.5-3.el7
+      - docker-ce-cli: 1:20.10.5-3.el7
+      - docker-ce-rootless-extras: 20.10.5-3.el7
     - hold: True
     - update_holds: True
 {% endif %}
@@ -232,7 +235,40 @@ commonlogrotateconf:
     - month: '*'
     - dayweek: '*'
 
+# Create the status directory
+sostatusdir:
+  file.directory:
+    - name: /opt/so/log/sostatus
+    - user: 0
+    - group: 0
+    - makedirs: True
+
+sostatus_log:
+  file.managed:
+    - name: /opt/so/log/sostatus/status.log
+    - mode: 644
+    
+# Install sostatus check cron
+'/usr/sbin/so-status -q; echo $? > /opt/so/log/sostatus/status.log 2>&1':
+  cron.present:
+    - user: root
+    - minute: '*/1'
+    - hour: '*'
+    - daymonth: '*'
+    - month: '*'
+    - dayweek: '*'
+
+
 {% if role in ['eval', 'manager', 'managersearch', 'standalone'] %}
+# Lock permissions on the backup directory
+backupdir:
+  file.directory:
+    - name: /nsm/backup
+    - user: 0
+    - group: 0
+    - makedirs: True
+    - mode: 700
+  
 # Add config backup
 /usr/sbin/so-config-backup > /dev/null 2>&1:
   cron.present:
@@ -242,6 +278,14 @@ commonlogrotateconf:
     - daymonth: '*'
     - month: '*'
     - dayweek: '*'
+{% else %}
+soversionfile:
+  file.managed:
+    - name: /etc/soversion
+    - source: salt://common/files/soversion
+    - mode: 644
+    - template: jinja
+    
 {% endif %}
 
 # Manager daemon.json
@@ -258,10 +302,35 @@ docker:
     - watch:
       - file: docker_daemon
 
-{% else %}
+# Reserve OS ports for Docker proxy in case boot settings are not already applied/present
+# 55000 = Wazuh, 57314 = Strelka, 47760-47860 = Zeek
+dockerapplyports:
+    cmd.run:
+      - name: if [ ! -s /etc/sysctl.d/99-reserved-ports.conf ]; then sysctl -w net.ipv4.ip_local_reserved_ports="55000,57314,47760-47860"; fi
 
-common_state_not_allowed:
-  test.fail_without_changes:
-    - name: common_state_not_allowed
+# Reserve OS ports for Docker proxy
+dockerreserveports:
+  file.managed:
+    - source: salt://common/files/99-reserved-ports.conf
+    - name: /etc/sysctl.d/99-reserved-ports.conf
+
+{% if salt['grains.get']('sosmodel', '') %}
+# Install raid check cron
+/usr/sbin/so-raid-status > /dev/null 2>&1:
+  cron.present:
+    - user: root
+    - minute: '*/15'
+    - hour: '*'
+    - daymonth: '*'
+    - month: '*'
+    - dayweek: '*'
+
+{% endif %}
+
+{% else %}
+
+{{sls}}_state_not_allowed:
+  test.fail_without_changes:
+    - name: {{sls}}_state_not_allowed
 
 {% endif %}

salt/common/tools/sbin/so-airgap-hotfixapply

@@ -0,0 +1,64 @@
+#!/bin/bash
+
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+. /usr/sbin/so-common
+
+UPDATE_DIR=/tmp/sohotfixapply
+
+if [ -z "$1" ]; then
+  echo "No tarball given. Please provide the filename so I can run the hotfix"
+  echo "so-airgap-hotfixapply /path/to/sohotfix.tar" 
+  exit 1
+else
+  if [ ! -f "$1" ]; then
+    echo "Unable to find $1. Make sure your path is correct and retry."
+    exit 1
+  else
+    echo "Determining if we need to apply this hotfix"
+    rm -rf $UPDATE_DIR
+    mkdir -p $UPDATE_DIR
+    tar xvf $1 -C $UPDATE_DIR
+
+    # Compare some versions
+    NEWVERSION=$(cat $UPDATE_DIR/VERSION)
+    HOTFIXVERSION=$(cat $UPDATE_DIR/HOTFIX)
+    CURRENTHOTFIX=$(cat /etc/sohotfix)
+    INSTALLEDVERSION=$(cat /etc/soversion)
+
+    if [ "$INSTALLEDVERSION" == "$NEWVERSION" ]; then
+      echo "Checking to see if there are hotfixes needed"
+      if [ "$HOTFIXVERSION" == "$CURRENTHOTFIX" ]; then
+        echo "You are already running the latest version of Security Onion."
+        rm -rf $UPDATE_DIR
+        exit 1
+      else
+        echo "We need to apply a hotfix"
+        copy_new_files
+        echo $HOTFIXVERSION > /etc/sohotfix
+        salt-call state.highstate -l info queue=True
+        echo "The Hotfix $HOTFIXVERSION has been applied"
+        # Clean up 
+        rm -rf $UPDATE_DIR
+        exit 0
+      fi
+    else
+      echo "This hotfix is not compatible with your current version. Download the latest ISO and run soup"
+      rm -rf $UPDATE_DIR
+    fi
+
+  fi
+fi

salt/common/tools/sbin/so-airgap-hotfixdownload

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# Get the latest code
+rm -rf /tmp/sohotfix
+mkdir -p /tmp/sohotfix
+cd /tmp/sohotfix
+git clone https://github.com/Security-Onion-Solutions/securityonion
+if [ ! -d "/tmp/sohotfix/securityonion" ]; then
+  echo "I was unable to get the latest code. Check your internet and try again."
+  exit 1
+else
+  echo "Looks like we have the code lets create the tarball."
+  cd /tmp/sohotfix/securityonion
+  tar cvf /tmp/sohotfix/sohotfix.tar HOTFIX VERSION salt pillar
+  echo ""
+  echo "Copy /tmp/sohotfix/sohotfix.tar to portable media and then copy it to your airgap manager."
+  exit 0
+fi

salt/common/tools/sbin/so-allow

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

salt/common/tools/sbin/so-allow-view

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

salt/common/tools/sbin/so-analyst-install

@@ -84,7 +84,7 @@ while [[ $INSTALL != "yes" ]] && [[ $INSTALL != "no" ]]; do
     echo "##                                       ##"
     echo "##    Installing the Security Onion      ##"
     echo "##   analyst node on this device will    ##"
-    echo "##      make permanenet changes to       ##"
+    echo "##       make permanent changes to       ##"
     echo "##              the system.              ##"
     echo "##                                       ##"
     echo "###########################################"

salt/common/tools/sbin/so-checkin

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

salt/common/tools/sbin/so-common

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -15,6 +15,8 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+DEFAULT_SALT_DIR=/opt/so/saltstack/default
+
 # Check for prerequisites
 if [ "$(id -u)" -ne 0 ]; then
 	echo "This script must be run using sudo!"
@@ -24,9 +26,226 @@ fi
 # Define a banner to separate sections
 banner="========================================================================="
 
+add_interface_bond0() {
+	local BNIC=$1
+	if [[ -z $MTU ]]; then
+		local MTU
+		MTU=$(lookup_pillar "mtu" "sensor")
+	fi
+	local nic_error=0
+
+	# Check if specific offload features are able to be disabled
+	for string in "generic-segmentation-offload" "generic-receive-offload" "tcp-segmentation-offload"; do
+		if ethtool -k "$BNIC" | grep $string | grep -q "on [fixed]"; then
+			echo "The hardware or driver for interface ${BNIC} is not supported, packet capture may not work as expected."
+			((nic_error++))
+			break
+		fi
+	done
+
+	case "$2" in
+		-v|--verbose)
+			local verbose=true
+		;;
+	esac
+
+	for i in rx tx sg tso ufo gso gro lro; do
+		if [[ $verbose == true ]]; then
+			ethtool -K "$BNIC" $i off
+		else
+			ethtool -K "$BNIC" $i off &>/dev/null
+		fi
+	done
+	# Check if the bond slave connection has already been created
+	nmcli -f name,uuid -p con | grep -q "bond0-slave-$BNIC"
+	local found_int=$?
+
+	if [[ $found_int != 0 ]]; then
+		# Create the slave interface and assign it to the bond
+		nmcli con add type ethernet ifname "$BNIC" con-name "bond0-slave-$BNIC" master bond0 -- \
+			ethernet.mtu "$MTU" \
+			connection.autoconnect "yes"
+	else
+		local int_uuid
+		int_uuid=$(nmcli -f name,uuid -p con | sed -n "s/bond0-slave-$BNIC //p" | tr -d ' ')
+
+		nmcli con mod "$int_uuid" \
+			ethernet.mtu "$MTU" \
+			connection.autoconnect "yes"
+	fi
+
+	ip link set dev "$BNIC" arp off multicast off allmulticast off promisc on
+			
+	# Bring the slave interface up
+	if [[ $verbose == true ]]; then
+		nmcli con up "bond0-slave-$BNIC"
+	else
+		nmcli con up "bond0-slave-$BNIC" &>/dev/null
+	fi
+	 
+	if [ "$nic_error" != 0 ]; then
+		return "$nic_error"
+	fi
+}
+
+check_airgap() {
+  # See if this is an airgap install
+  AIRGAP=$(cat /opt/so/saltstack/local/pillar/global.sls | grep airgap: | awk '{print $2}')
+  if [[ "$AIRGAP" == "True" ]]; then
+      is_airgap=0
+      UPDATE_DIR=/tmp/soagupdate/SecurityOnion
+      AGDOCKER=/tmp/soagupdate/docker
+      AGREPO=/tmp/soagupdate/Packages
+  else 
+      is_airgap=1
+  fi
+}
+
+check_container() {
+	docker ps | grep "$1:" > /dev/null 2>&1
+	return $?
+}
+
+check_password() {
+	local password=$1
+	echo "$password" | egrep -v "'|\"|\\$|\\\\" > /dev/null 2>&1
+	return $?
+}
+
+check_elastic_license() {
+
+	[ -n "$TESTING" ] && return
+
+	# See if the user has already accepted the license
+	if [ ! -f /opt/so/state/yeselastic.txt ]; then
+	  elastic_license
+	else
+	  echo "Elastic License has already been accepted"
+	fi  
+}
+
+copy_new_files() {
+  # Copy new files over to the salt dir
+  cd $UPDATE_DIR
+  rsync -a salt $DEFAULT_SALT_DIR/
+  rsync -a pillar $DEFAULT_SALT_DIR/
+  chown -R socore:socore $DEFAULT_SALT_DIR/
+  chmod 755 $DEFAULT_SALT_DIR/pillar/firewall/addfirewall.sh
+  cd /tmp
+}
+
+disable_fastestmirror() {
+	sed -i 's/enabled=1/enabled=0/' /etc/yum/pluginconf.d/fastestmirror.conf
+}
+
+elastic_license() {
+
+read -r -d '' message <<- EOM
+\n
+Starting in Elastic Stack version 7.11, the Elastic Stack binaries are only available under the Elastic License:
+https://securityonion.net/elastic-license
+
+Please review the Elastic License:
+https://www.elastic.co/licensing/elastic-license
+
+Do you agree to the terms of the Elastic License?
+
+If so, type AGREE to accept the Elastic License and continue.  Otherwise, press Enter to exit this program without making any changes.
+EOM
+
+AGREED=$(whiptail --title "Security Onion Setup" --inputbox \
+"$message" 20 75 3>&1 1>&2 2>&3)
+
+if [ "${AGREED^^}" = 'AGREE' ]; then
+  mkdir -p /opt/so/state
+  touch /opt/so/state/yeselastic.txt 
+else
+  echo "Starting in 2.3.40 you must accept the Elastic license if you want to run Security Onion."
+  exit 1
+fi
+
+}
+
+fail() {
+	msg=$1
+	echo "ERROR: $msg"
+	echo "Exiting."
+	exit 1
+}
+
+get_random_value() {
+	length=${1:-20}
+	head -c 5000 /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w $length | head -n 1
+}
+
+gpg_rpm_import() {
+	if [[ "$OS" == "centos" ]]; then
+	    if [[ "$WHATWOULDYOUSAYYAHDOHERE" == "setup" ]]; then
+	        local RPMKEYSLOC="../salt/repo/client/files/centos/keys"
+	    else
+	        local RPMKEYSLOC="$UPDATE_DIR/salt/repo/client/files/centos/keys"
+	    fi
+	
+	    RPMKEYS=('RPM-GPG-KEY-EPEL-7' 'GPG-KEY-WAZUH' 'docker.pub' 'SALTSTACK-GPG-KEY.pub' 'securityonion.pub')
+
+	    for RPMKEY in "${RPMKEYS[@]}"; do
+    	        rpm --import $RPMKEYSLOC/$RPMKEY
+	        echo "Imported $RPMKEY"
+	    done
+	fi
+}
+
 header() {
-	echo
-    printf '%s\n' "$banner" "$*" "$banner"
+	printf '%s\n' "" "$banner" " $*" "$banner" 
+}
+
+init_monitor() {
+	MONITORNIC=$1
+	
+	if [[ $MONITORNIC == "bond0" ]]; then 
+	  BIFACES=$(lookup_bond_interfaces)
+	else
+	  BIFACES=$MONITORNIC
+	fi
+
+	for DEVICE_IFACE in $BIFACES; do
+	  for i in rx tx sg tso ufo gso gro lro; do
+	    ethtool -K "$DEVICE_IFACE" "$i" off;
+  	  done
+	  ip link set dev "$DEVICE_IFACE" arp off multicast off allmulticast off promisc on
+	done
+}
+
+is_manager_node() {
+	# Check to see if this is a manager node
+	role=$(lookup_role)
+	is_single_node_grid && return 0
+	[ $role == 'manager' ] && return 0
+	[ $role == 'managersearch' ] && return 0
+	[ $role == 'helix' ] && return 0
+	return 1
+}
+
+is_sensor_node() {
+	# Check to see if this is a sensor (forward) node
+	role=$(lookup_role)
+	is_single_node_grid && return 0
+	[ $role == 'sensor' ] && return 0
+	[ $role == 'heavynode' ] && return 0
+	[ $role == 'helix' ] && return 0
+	return 1
+}
+
+is_single_node_grid() {
+	role=$(lookup_role)
+	[ $role == 'eval' ] && return 0
+	[ $role == 'standalone' ] && return 0
+	[ $role == 'import' ] && return 0
+	return 1
+}
+
+lookup_bond_interfaces() {
+	cat /proc/net/bonding/bond0 | grep "Slave Interface:" | sed -e "s/Slave Interface: //g"
 }
 
 lookup_salt_value() {
@@ -68,15 +287,41 @@ lookup_role() {
 	echo ${pieces[1]}
 }
 
-check_container() {
-	docker ps | grep "$1:" > /dev/null 2>&1
-	return $?
+require_manager() {
+	if is_manager_node; then
+		echo "This is a manager, We can proceed."
+	else
+		echo "Please run this command on the manager; the manager controls the grid."
+		exit 1
+	fi
 }
 
-check_password() {
-  local password=$1
-  echo "$password" | egrep -v "'|\"|\\$|\\\\" > /dev/null 2>&1
-  return $?
+retry() {
+	maxAttempts=$1
+	sleepDelay=$2
+	cmd=$3
+	expectedOutput=$4
+	attempt=0
+	while [[ $attempt -lt $maxAttempts ]]; do			
+		attempt=$((attempt+1))
+		echo "Executing command with retry support: $cmd"
+		output=$(eval "$cmd")
+		exitcode=$?
+		echo "Results: $output ($exitcode)"
+		if [ -n "$expectedOutput" ]; then
+			if [[ "$output" =~ "$expectedOutput" ]]; then
+				return $exitCode
+			else
+				echo "Expected '$expectedOutput' but got '$output'"
+			fi
+		elif [[ $exitcode -eq 0 ]]; then
+			return $exitCode
+		fi
+		echo "Command failed with exit code $exitcode; will retry in $sleepDelay seconds ($attempt / $maxAttempts)..."
+		sleep $sleepDelay
+	done
+	echo "Command continues to fail; giving up."
+	return 1
 }
 
 set_os() {
@@ -91,6 +336,12 @@ set_minionid() {
 	MINIONID=$(lookup_grain id)
 }
 
+set_palette() {
+	if [ "$OS" == ubuntu ]; then
+	    update-alternatives --set newt-palette /etc/newt/palette.original
+    fi
+}
+
 set_version() {
 	CURRENTVERSION=0.0.0
 	if [ -f /etc/soversion ]; then
@@ -110,33 +361,153 @@ set_version() {
 	fi
 }
 
-require_manager() {
-  # Check to see if this is a manager
-  MANAGERCHECK=$(cat /etc/salt/grains | grep role | awk '{print $2}')
-  if [ $MANAGERCHECK == 'so-eval' ] || [ $MANAGERCHECK == 'so-manager' ] || [ $MANAGERCHECK == 'so-managersearch' ] || [ $MANAGERCHECK == 'so-standalone' ] || [ $MANAGERCHECK == 'so-helix' ] || [ $MANAGERCHECK == 'so-import' ]; then
-    echo "This is a manager, We can proceed."
+valid_cidr() {
+	# Verify there is a backslash in the string
+	echo "$1" | grep -qP "^[^/]+/[^/]+$" || return 1
+	
+	local cidr
+	local ip
+
+	cidr=$(echo "$1" | sed 's/.*\///')
+	ip=$(echo "$1" | sed 's/\/.*//' )
+	
+	if valid_ip4 "$ip"; then
+		[[ $cidr =~ ([0-9]|[1-2][0-9]|3[0-2]) ]] && return 0 || return 1
 	else
-    echo "Please run this command on the manager; the manager controls the grid."
-    exit 1
+		return 1
 	fi
 }
 
-is_single_node_grid() {
-  role=$(lookup_role)
-  if [ "$role" != "eval" ] && [ "$role" != "standalone" ] && [ "$role" != "import" ]; then
+valid_cidr_list() {
+	local all_valid=0
+
+	IFS="," read -r -a net_arr <<< "$1"
+
+	for net in "${net_arr[@]}"; do
+		valid_cidr "$net" || all_valid=1
+	done
+
+	return $all_valid
+}
+
+valid_dns_list() {
+	local all_valid=0
+
+	IFS="," read -r -a dns_arr <<< "$1"
+
+	for addr in "${dns_arr[@]}"; do
+		valid_ip4 "$addr" || all_valid=1
+	done
+
+	return $all_valid
+}
+
+valid_fqdn() {
+	local fqdn=$1
+
+	echo "$fqdn" | grep -qP '(?=^.{4,253}$)(^((?!-)[a-zA-Z0-9-]{0,62}[a-zA-Z0-9]\.)+[a-zA-Z]{2,63}$)' \
+		&& return 0 \
+		|| return 1
+}
+
+valid_hostname() {
+	local hostname=$1
+
+	[[ $hostname =~ ^[a-zA-Z0-9\-]+$ ]] && [[ $hostname != 'localhost' ]] && return 0 || return 1
+}
+
+valid_ip4() {
+	local ip=$1
+
+	echo "$ip" | grep -qP '^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$' && return 0 || return 1
+}
+
+valid_int() {
+	local num=$1
+	local min=${2:-1}
+	local max=${3:-1000000000}
+
+	[[ $num =~ ^[0-9]*$ ]] && [[ $num -ge $min ]] && [[ $num -le $max ]] && return 0 || return 1
+}
+
+# {% raw %}
+
+valid_proxy() {
+	local proxy=$1
+	local url_prefixes=( 'http://' 'https://' )
+
+	local has_prefix=false
+	for prefix in "${url_prefixes[@]}"; do
+		echo "$proxy" | grep -q "$prefix" && has_prefix=true && proxy=${proxy#"$prefix"} && break
+	done
+	
+	local url_arr
+	mapfile -t url_arr <<< "$(echo "$proxy" | tr ":" "\n")"
+
+	local valid_url=true
+	if ! valid_ip4 "${url_arr[0]}" && ! valid_fqdn "${url_arr[0]}" && ! valid_hostname "${url_arr[0]}"; then
+		valid_url=false
+	fi
+
+	[[ $has_prefix == true ]] && [[ $valid_url == true ]] && return 0 || return 1
+}
+
+valid_ntp_list() {
+	local string=$1
+	local ntp_arr
+	IFS="," read -r -a ntp_arr <<< "$string"
+
+	for ntp in "${ntp_arr[@]}"; do
+		if ! valid_ip4 "$ntp" && ! valid_hostname "$ntp" && ! valid_fqdn "$ntp"; then
 			return 1
 		fi
+	done
+
 	return 0
 }
 
-fail() {
-  msg=$1
-  echo "ERROR: $msg"
-  echo "Exiting."
-  exit 1
+valid_string() {
+	local str=$1
+	local min_length=${2:-1}
+	local max_length=${3:-64}
+
+	echo "$str" | grep -qP '^\S+$' && [[ ${#str} -ge $min_length ]] && [[ ${#str} -le $max_length ]] && return 0 || return 1
 }
 
-get_random_value() {
-  length=${1:-20}
-  head -c 5000 /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w $length | head -n 1
+# {% endraw %}
+
+valid_username() {
+	local user=$1
+
+	echo "$user" | grep -qP '^[a-z_]([a-z0-9_-]{0,31}|[a-z0-9_-]{0,30}\$)$' && return 0 || return 1
+}
+
+wait_for_web_response() {
+	url=$1
+	expected=$2
+	maxAttempts=${3:-300}
+	logfile=/root/wait_for_web_response.log
+	truncate -s 0 "$logfile"
+	attempt=0
+	while [[ $attempt -lt $maxAttempts ]]; do
+		attempt=$((attempt+1))
+		echo "Waiting for value '$expected' at '$url' ($attempt/$maxAttempts)"
+		result=$(curl -ks -L $url)
+		exitcode=$?
+
+		echo "--------------------------------------------------" >> $logfile
+		echo "$(date) - Checking web URL: $url ($attempt/$maxAttempts)" >> $logfile
+		echo "$result" >> $logfile
+		echo "exit code=$exitcode" >> $logfile
+		echo "" >> $logfile
+
+		if [[ $exitcode -eq 0 && "$result" =~ $expected ]]; then
+			echo "Received expected response; proceeding."
+			return 0
+		fi
+		echo "Server is not ready"
+		sleep 1
+	done
+	echo "Server still not ready after $maxAttempts attempts; giving up."
+	return 1
 }

salt/common/tools/sbin/so-config-backup

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -33,12 +33,15 @@ if [ ! -f $BACKUPFILE ]; then
   {%- for LOCATION in BACKUPLOCATIONS %}
   tar -rf $BACKUPFILE {{ LOCATION }}
   {%- endfor %}
+  tar -rf $BACKUPFILE /etc/pki
+  tar -rf $BACKUPFILE /etc/salt
 
 fi
 
-# Find oldest backup file and remove it
+# Find oldest backup files and remove them
 NUMBACKUPS=$(find /nsm/backup/ -type f -name "so-config-backup*" | wc -l)
-OLDESTBACKUP=$(find /nsm/backup/ -type f -name "so-config-backup*" | ls -1t | tail -1)
-if [ "$NUMBACKUPS" -gt "$MAXBACKUPS" ]; then
-  rm -f /nsm/backup/$OLDESTBACKUP
-fi
+while [ "$NUMBACKUPS" -gt "$MAXBACKUPS" ]; do
+  OLDESTBACKUP=$(find /nsm/backup/ -type f -name "so-config-backup*" -type f -printf '%T+ %p\n' | sort | head -n 1 | awk -F" " '{print $2}')
+  rm -f $OLDESTBACKUP
+  NUMBACKUPS=$(find /nsm/backup/ -type f -name "so-config-backup*" | wc -l)
+done

salt/common/tools/sbin/so-cortex-restart

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

salt/common/tools/sbin/so-cortex-start

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

salt/common/tools/sbin/so-cortex-stop

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

salt/common/tools/sbin/so-cortex-user-add

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -30,7 +30,7 @@ fi
 
 USER=$1
 
-CORTEX_KEY=$(lookup_pillar cortexkey)
+CORTEX_KEY=$(lookup_pillar cortexorguserkey)
 CORTEX_API_URL="$(lookup_pillar url_base)/cortex/api"
 CORTEX_ORG_NAME=$(lookup_pillar cortexorgname)
 CORTEX_USER=$USER

salt/common/tools/sbin/so-cortex-user-enable

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -30,7 +30,7 @@ fi
 
 USER=$1
 
-CORTEX_KEY=$(lookup_pillar cortexkey)
+CORTEX_KEY=$(lookup_pillar cortexorguserkey)
 CORTEX_API_URL="$(lookup_pillar url_base)/cortex/api"
 CORTEX_USER=$USER
 

salt/common/tools/sbin/so-curator-restart

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

salt/common/tools/sbin/so-curator-start

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

salt/common/tools/sbin/so-curator-stop

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

salt/common/tools/sbin/so-docker-prune

@@ -0,0 +1,89 @@
+#!/usr/bin/env python3
+
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import sys, argparse, re, docker
+from packaging.version import Version, InvalidVersion
+from itertools import groupby, chain
+
+
+def get_image_name(string) -> str:
+  return ':'.join(string.split(':')[:-1])
+
+
+def get_so_image_basename(string) -> str:
+  return get_image_name(string).split('/so-')[-1]
+
+
+def get_image_version(string) -> str:
+  ver = string.split(':')[-1]
+  if ver == 'latest':
+    # Version doesn't like "latest", so use a high semver
+    return '999999.9.9'
+  else:
+    try:
+      Version(ver)
+    except InvalidVersion:
+      # Strip the last substring following a hyphen for automated branches
+      ver = '-'.join(ver.split('-')[:-1])  
+    return ver
+
+
+def main(quiet):
+  client = docker.from_env()
+
+  image_list = client.images.list(filters={ 'dangling': False })
+
+  # Map list of image objects to flattened list of tags (format: "name:version")
+  tag_list = list(chain.from_iterable(list(map(lambda x: x.attrs.get('RepoTags'), image_list))))
+
+  # Filter to only SO images (base name begins with "so-")
+  tag_list = list(filter(lambda x: re.match(r'^.*\/so-[^\/]*$', get_image_name(x)), tag_list))
+
+  # Group tags into lists by base name (sort by same projection first)
+  tag_list.sort(key=lambda x: get_so_image_basename(x))
+  grouped_tag_lists = [ list(it) for _, it in groupby(tag_list, lambda x: get_so_image_basename(x)) ]
+
+  no_prunable = True
+  for t_list in grouped_tag_lists:
+    try:
+      # Group tags by version, in case multiple images exist with the same version string
+      t_list.sort(key=lambda x: Version(get_image_version(x)), reverse=True)
+      grouped_t_list = [ list(it) for _,it in groupby(t_list, lambda x: get_image_version(x)) ]
+
+      # Keep the 2 most current version groups
+      if len(grouped_t_list) <= 2:
+        continue
+      else:
+        no_prunable = False
+        for group in grouped_t_list[2:]:
+          for tag in group:
+            if not quiet: print(f'Removing image {tag}')
+            client.images.remove(tag)
+    except InvalidVersion as e:
+      print(f'so-{get_so_image_basename(t_list[0])}: {e.args[0]}', file=sys.stderr)
+      exit(1)
+
+  if no_prunable and not quiet:
+    print('No Security Onion images to prune')
+
+
+if __name__ == "__main__":
+  main_parser = argparse.ArgumentParser(add_help=False)
+  main_parser.add_argument('-q', '--quiet', action='store_const', const=True, required=False)
+  args = main_parser.parse_args(sys.argv[1:])
+ 
+  main(args.quiet)

salt/common/tools/sbin/so-docker-refresh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

salt/common/tools/sbin/so-elastalert-restart

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

salt/common/tools/sbin/so-elastalert-start

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

salt/common/tools/sbin/so-elastalert-stop

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

salt/common/tools/sbin/so-elastalert-test

@@ -96,7 +96,7 @@ rule_prompt(){
 	echo "-----------------------------------"
 	echo
 	while [ -z "$RULE_NAME" ]; do
-		read -p "Please enter the rule filename you want to test (filename only, no path): " -e RULE_NAME
+		read -p "Choose a rule to test from the list above (must be typed exactly as shown above): " -e RULE_NAME
 	done
 }
 

salt/common/tools/sbin/so-elastic-clear

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -50,11 +50,7 @@ done
 if [ $SKIP -ne 1 ]; then
         # List indices
         echo 
-        {% if grains['role'] in ['so-node','so-heavynode'] %}
         curl -k -L https://{{ NODEIP }}:9200/_cat/indices?v
-        {% else %}
-        curl -L {{ NODEIP }}:9200/_cat/indices?v
-        {% endif %}
         echo
         # Inform user we are about to delete all data
         echo
@@ -93,18 +89,10 @@ fi
 # Delete data
 echo "Deleting data..."
 
-{% if grains['role'] in ['so-node','so-heavynode'] %}
 INDXS=$(curl -s -XGET -k -L https://{{ NODEIP }}:9200/_cat/indices?v | egrep 'logstash|elastalert|so-' | awk '{ print $3 }')
-{% else %}
-INDXS=$(curl -s -XGET -L {{ NODEIP }}:9200/_cat/indices?v | egrep 'logstash|elastalert|so-' | awk '{ print $3 }')
-{% endif %}
 for INDX in ${INDXS}
 do
-    {% if grains['role'] in ['so-node','so-heavynode'] %}
     curl -XDELETE -k -L https://"{{ NODEIP }}:9200/${INDX}" > /dev/null 2>&1
-    {% else %}
-    curl -XDELETE -L "{{ NODEIP }}:9200/${INDX}" > /dev/null 2>&1
-    {% endif %}
 done
 
 #Start Logstash/Filebeat

salt/common/tools/sbin/so-elastic-diagnose

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

salt/common/tools/sbin/so-elastic-restart

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

salt/common/tools/sbin/so-elastic-start

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

salt/common/tools/sbin/so-elastic-stop

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

salt/common/tools/sbin/so-elasticsearch-indices-list

@@ -0,0 +1,21 @@
+#!/bin/bash
+#
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>
+{%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%}
+
+. /usr/sbin/so-common
+
+curl -s -k -L https://{{ NODEIP }}:9200/_cat/indices?pretty

salt/common/tools/sbin/so-elasticsearch-indices-rw

@@ -1,7 +1,7 @@
 #!/bin/bash
 #
 #
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -21,6 +21,5 @@ THEHIVEESPORT=9400
 
 echo "Removing read only attributes for indices..."
 echo
-for p in $ESPORT $THEHIVEESPORT; do
-   curl -XPUT -H "Content-Type: application/json" -L http://$IP:$p/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}' 2>&1 | if grep -q ack; then echo "Index settings updated..."; else echo "There was any issue updating the read-only attribute.  Please ensure Elasticsearch is running.";fi;
-done
+curl -s -k -XPUT -H "Content-Type: application/json" -L https://$IP:9200/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}' 2>&1 | if grep -q ack; then echo "Index settings updated..."; else echo "There was any issue updating the read-only attribute.  Please ensure Elasticsearch is running.";fi;
+curl -XPUT -H "Content-Type: application/json" -L http://$IP:9400/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}' 2>&1 | if grep -q ack; then echo "Index settings updated..."; else echo "There was any issue updating the read-only attribute.  Please ensure Elasticsearch is running.";fi;

salt/common/tools/sbin/so-elasticsearch-pipeline-stats

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -19,15 +19,7 @@
 . /usr/sbin/so-common
 
 if [ "$1" == "" ]; then
-        {% if grains['role'] in ['so-node','so-heavynode'] %}
         curl -s -k -L https://{{ NODEIP }}:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines"
-        {% else %}
-        curl -s -L {{ NODEIP }}:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines"
-        {% endif %}
 else
-        {% if grains['role'] in ['so-node','so-heavynode'] %}
         curl -s -k -L https://{{ NODEIP }}:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines.\"$1\""
-        {% else %}
-        curl -s -L {{ NODEIP }}:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines.\"$1\""
-        {% endif %}
 fi

salt/common/tools/sbin/so-elasticsearch-pipeline-view

@@ -0,0 +1,25 @@
+#!/bin/bash
+#
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>
+{%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%}
+
+. /usr/sbin/so-common
+
+if [ "$1" == "" ]; then
+        curl -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/* | jq .
+else
+        curl -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/$1 | jq .
+fi

salt/common/tools/sbin/so-elasticsearch-pipelines-list

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -17,15 +17,7 @@
 {%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%}
 . /usr/sbin/so-common
 if [ "$1" == "" ]; then
-    {% if grains['role'] in ['so-node','so-heavynode'] %}
     curl -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/* | jq 'keys'
-    {% else %}
-    curl -s -L {{ NODEIP }}:9200/_ingest/pipeline/* | jq 'keys'
-    {% endif %}
 else
-    {% if grains['role'] in ['so-node','so-heavynode'] %}
     curl -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/$1 | jq
-    {% else %}
-    curl -s -L {{ NODEIP }}:9200/_ingest/pipeline/$1 | jq
-    {% endif %}
 fi

salt/common/tools/sbin/so-elasticsearch-restart

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

salt/common/tools/sbin/so-elasticsearch-shards-list

@@ -0,0 +1,21 @@
+#!/bin/bash
+#
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>
+{%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%}
+
+. /usr/sbin/so-common
+
+curl -s -k -L https://{{ NODEIP }}:9200/_cat/shards?pretty

salt/common/tools/sbin/so-elasticsearch-start

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

salt/common/tools/sbin/so-elasticsearch-stop

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

salt/common/tools/sbin/so-elasticsearch-template-remove

@@ -0,0 +1,21 @@
+#!/bin/bash
+#
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>
+{%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%}
+
+. /usr/sbin/so-common
+
+curl -s -k -L -XDELETE https://{{ NODEIP }}:9200/_template/$1

salt/common/tools/sbin/so-elasticsearch-template-view

@@ -0,0 +1,25 @@
+#!/bin/bash
+#
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>
+{%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%}
+
+. /usr/sbin/so-common
+
+if [ "$1" == "" ]; then
+        curl -s -k -L https://{{ NODEIP }}:9200/_template/* | jq .
+else
+        curl -s -k -L https://{{ NODEIP }}:9200/_template/$1 | jq .
+fi

salt/common/tools/sbin/so-elasticsearch-templates-list

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -17,15 +17,7 @@
 {%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%}
 . /usr/sbin/so-common
 if [ "$1" == "" ]; then
-    {% if grains['role'] in ['so-node','so-heavynode'] %}
     curl -s -k -L https://{{ NODEIP }}:9200/_template/* | jq 'keys'
-    {% else %}
-    curl -s -L {{ NODEIP }}:9200/_template/* | jq 'keys'
-    {% endif %}
 else
-    {% if grains['role'] in ['so-node','so-heavynode'] %}
     curl -s -k -L https://{{ NODEIP }}:9200/_template/$1 | jq
-    {% else %}
-    curl -s -L {{ NODEIP }}:9200/_template/$1 | jq
-    {% endif %}
 fi

salt/common/tools/sbin/so-elasticsearch-templates-load

@@ -30,11 +30,7 @@ echo -n "Waiting for ElasticSearch..."
 COUNT=0
 ELASTICSEARCH_CONNECTED="no"
 while [[ "$COUNT" -le 240 ]]; do
-    {% if grains['role'] in ['so-node','so-heavynode'] %}
     curl -k --output /dev/null --silent --head --fail -L https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"
-    {% else %}
-	curl --output /dev/null --silent --head --fail -L http://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"
-	{% endif %}
 	if [ $? -eq 0 ]; then
 		ELASTICSEARCH_CONNECTED="yes"
 		echo "connected!"
@@ -55,11 +51,7 @@ cd ${ELASTICSEARCH_TEMPLATES}
 
 
 echo "Loading templates..."
-{% if grains['role'] in ['so-node','so-heavynode'] %}
 for i in *; do TEMPLATE=$(echo $i | cut -d '-' -f2); echo "so-$TEMPLATE"; curl -k ${ELASTICSEARCH_AUTH} -s -XPUT -L https://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_template/so-$TEMPLATE -H 'Content-Type: application/json' -d@$i 2>/dev/null; echo; done
-{% else %}
-for i in *; do TEMPLATE=$(echo $i | cut -d '-' -f2); echo "so-$TEMPLATE"; curl ${ELASTICSEARCH_AUTH} -s -XPUT -L http://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_template/so-$TEMPLATE -H 'Content-Type: application/json' -d@$i 2>/dev/null; echo; done
-{% endif %}
 echo
 
 cd - >/dev/null

salt/common/tools/sbin/so-features-enable

@@ -1,53 +0,0 @@
-#!/bin/bash
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
-
-#    This program is free software: you can redistribute it and/or modify
-#    it under the terms of the GNU General Public License as published by
-#    the Free Software Foundation, either version 3 of the License, or
-#    (at your option) any later version.
-#
-#    This program is distributed in the hope that it will be useful,
-#    but WITHOUT ANY WARRANTY; without even the implied warranty of
-#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#    GNU General Public License for more details.
-#
-#    You should have received a copy of the GNU General Public License
-#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-. /usr/sbin/so-common
-. /usr/sbin/so-image-common
-local_salt_dir=/opt/so/saltstack/local
-
-cat << EOF
-This program will switch from the open source version of the Elastic Stack to the Features version licensed under the Elastic license.
-If you proceed, then we will download new Docker images and restart services.
-
-Please review the Elastic license:
-https://raw.githubusercontent.com/elastic/elasticsearch/master/licenses/ELASTIC-LICENSE.txt
-
-Please also note that, if you have a distributed deployment and continue with this change, Elastic traffic between nodes will change from encrypted to cleartext!
-(We expect to support Elastic Features Security at some point in the future.)
-
-Do you agree to the terms of the Elastic license and understand the note about encryption?
-
-If so, type AGREE to accept the Elastic license and continue.  Otherwise, just press Enter to exit this program without making any changes.
-EOF
-
-read INPUT
-if [ "$INPUT" != "AGREE" ]; then
-        exit
-fi
-
-echo "Please wait while switching to Elastic Features."
-
-require_manager
-
-TRUSTED_CONTAINERS=( \
-  "so-elasticsearch" \
-  "so-filebeat" \
-  "so-kibana" \
-  "so-logstash" )
-update_docker_containers "features" "-features"
-
-# Modify global.sls to enable Features
-sed -i 's/features: False/features: True/' $local_salt_dir/pillar/global.sls

salt/common/tools/sbin/so-filebeat-restart

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

salt/common/tools/sbin/so-filebeat-start

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

salt/common/tools/sbin/so-filebeat-stop

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

salt/common/tools/sbin/so-firewall

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -15,27 +15,38 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import os
 import subprocess
 import sys
+import time
 import yaml
 
+lockFile = "/tmp/so-firewall.lock"
 hostgroupsFilename = "/opt/so/saltstack/local/salt/firewall/hostgroups.local.yaml"
 portgroupsFilename = "/opt/so/saltstack/local/salt/firewall/portgroups.local.yaml"
+defaultPortgroupsFilename = "/opt/so/saltstack/default/salt/firewall/portgroups.yaml"
 supportedProtocols = ['tcp', 'udp']
 
-def showUsage(args):
+def showUsage(options, args):
   print('Usage: {} [OPTIONS] <COMMAND> [ARGS...]'.format(sys.argv[0]))
   print('  Options:')
   print('   --apply         - After updating the firewall configuration files, apply the new firewall state')
+  print('   --defaultports  - Read port groups from default configuration files instead of local configuration.')
   print('')
-  print('  Available commands:')
+  print('  General commands:')
   print('    help           - Prints this usage information.')
+  print('')
+  print('  Host commands:')
+  print('    listhostgroups - Lists the known host groups.')
   print('    includedhosts  - Lists the IPs included in the given group. Args: <GROUP_NAME>')
   print('    excludedhosts  - Lists the IPs excluded from the given group. Args: <GROUP_NAME>')
   print('    includehost    - Includes the given IP in the given group. Args: <GROUP_NAME> <IP>')
   print('    excludehost    - Excludes the given IP from the given group. Args: <GROUP_NAME> <IP>')
   print('    removehost     - Removes an excluded IP from the given group. Args: <GROUP_NAME> <IP>')
   print('    addhostgroup   - Adds a new, custom host group. Args: <GROUP_NAME>')
+  print('')
+  print('  Port commands:')
+  print('    listportgroups - Lists the known port groups.')
   print('    listports      - Lists ports in the given group and protocol. Args: <GROUP_NAME> <PORT_PROTOCOL>')
   print('    addport        - Adds a PORT to the given group. Args: <GROUP_NAME> <PORT_PROTOCOL> <PORT>')
   print('    removeport     - Removes a PORT from the given group. Args: <GROUP_NAME> <PORT_PROTOCOL> <PORT>')
@@ -48,6 +59,15 @@ def showUsage(args):
   print('   PORT          - Either a single numeric port (Ex: 443), or a port range (Ex: 8000:8002).')
   sys.exit(1)
 
+def checkDefaultPortsOption(options):
+  global portgroupsFilename
+  if "--defaultports" in options:
+    portgroupsFilename = defaultPortgroupsFilename
+
+def checkApplyOption(options):
+  if "--apply" in options:
+    return apply()
+
 def loadYaml(filename):
   file = open(filename, "r")
   return yaml.load(file.read())
@@ -56,6 +76,14 @@ def writeYaml(filename, content):
   file = open(filename, "w")
   return yaml.dump(content, file)
 
+def listHostGroups():
+  content = loadYaml(hostgroupsFilename)
+  hostgroups = content['firewall']['hostgroups']
+  if hostgroups is not None:
+    for group in hostgroups:
+      print(group)
+  return 0
+
 def listIps(name, mode):
   content = loadYaml(hostgroupsFilename)
   if name not in content['firewall']['hostgroups']:
@@ -111,10 +139,18 @@ def createProtocolMap():
     map[protocol] = []
   return map
 
-def addhostgroup(args):
+def listPortGroups():
+  content = loadYaml(portgroupsFilename)
+  portgroups = content['firewall']['aliases']['ports']
+  if portgroups is not None:
+    for group in portgroups:
+      print(group)
+  return 0
+
+def addhostgroup(options, args):
   if len(args) != 1:
     print('Missing host group name argument', file=sys.stderr)
-    showUsage(args)
+    showUsage(options, args)
 
   name = args[0]
   content = loadYaml(hostgroupsFilename)
@@ -125,10 +161,17 @@ def addhostgroup(args):
   writeYaml(hostgroupsFilename, content)
   return 0
 
-def addportgroup(args):
+def listportgroups(options, args):
+  if len(args) != 0:
+    print('Unexpected arguments', file=sys.stderr)
+    showUsage(options, args)
+  checkDefaultPortsOption(options)
+  return listPortGroups()
+
+def addportgroup(options, args):
   if len(args) != 1:
     print('Missing port group name argument', file=sys.stderr)
-    showUsage(args)
+    showUsage(options, args)
 
   name = args[0]
   content = loadYaml(portgroupsFilename)
@@ -143,11 +186,12 @@ def addportgroup(args):
   writeYaml(portgroupsFilename, content)
   return 0
 
-def listports(args):
+def listports(options, args):
   if len(args) != 2:
     print('Missing port group name or port protocol', file=sys.stderr)
-    showUsage(args)
+    showUsage(options, args)
 
+  checkDefaultPortsOption(options)
   name = args[0]
   protocol = args[1]
   if protocol not in supportedProtocols:
@@ -162,16 +206,19 @@ def listports(args):
   if name not in ports:
     print('Port group does not exist', file=sys.stderr)
     return 3
+  if protocol not in ports[name]:
+    print('Port group does not contain protocol', file=sys.stderr)
+    return 3
   ports = ports[name][protocol]
   if ports is not None:
     for port in ports:
       print(port)
   return 0
 
-def addport(args):
+def addport(options, args):
   if len(args) != 3:
     print('Missing port group name or port protocol, or port argument', file=sys.stderr)
-    showUsage(args)
+    showUsage(options, args)
 
   name = args[0]
   protocol = args[1]
@@ -197,12 +244,13 @@ def addport(args):
     return 3
   ports.append(port)
   writeYaml(portgroupsFilename, content)
-  return 0
+  code = checkApplyOption(options)
+  return code
 
-def removeport(args):
+def removeport(options, args):
   if len(args) != 3:
     print('Missing port group name or port protocol, or port argument', file=sys.stderr)
-    showUsage(args)
+    showUsage(options, args)
 
   name = args[0]
   protocol = args[1]
@@ -225,43 +273,60 @@ def removeport(args):
     return 3
   ports.remove(port)
   writeYaml(portgroupsFilename, content)
-  return 0
+  code = checkApplyOption(options)
+  return code
 
-def includedhosts(args):
+
+def listhostgroups(options, args):
+  if len(args) != 0:
+    print('Unexpected arguments', file=sys.stderr)
+    showUsage(options, args)
+  return listHostGroups()
+
+def includedhosts(options, args):
   if len(args) != 1:
     print('Missing host group name argument', file=sys.stderr)
-    showUsage(args)
+    showUsage(options, args)
   return listIps(args[0], 'insert')
 
-def excludedhosts(args):
+def excludedhosts(options, args):
   if len(args) != 1:
     print('Missing host group name argument', file=sys.stderr)
-    showUsage(args)
+    showUsage(options, args)
   return listIps(args[0], 'delete')
 
-def includehost(args):
+def includehost(options, args):
   if len(args) != 2:
     print('Missing host group name or ip argument', file=sys.stderr)
-    showUsage(args)
+    showUsage(options, args)
   result = addIp(args[0], args[1], 'insert')
   if result == 0:
     removeIp(args[0], args[1], 'delete', True)
-  return result
+  code = result
+  if code == 0:
+    code = checkApplyOption(options)
+  return code
 
-def excludehost(args):
+def excludehost(options, args):
   if len(args) != 2:
     print('Missing host group name or ip argument', file=sys.stderr)
-    showUsage(args)
+    showUsage(options, args)
   result = addIp(args[0], args[1], 'delete')
   if result == 0:
     removeIp(args[0], args[1], 'insert', True)
-  return result
+  code = result
+  if code == 0:
+    code = checkApplyOption(options)
+  return code
 
-def removehost(args):
+def removehost(options, args):
   if len(args) != 2:
     print('Missing host group name or ip argument', file=sys.stderr)
-    showUsage(args)
-  return removeIp(args[0], args[1], 'delete')
+    showUsage(options, args)
+  code = removeIp(args[0], args[1], 'delete')
+  if code == 0:
+    code = checkApplyOption(options)
+  return code
 
 def apply():
   proc = subprocess.run(['salt-call', 'state.apply', 'firewall', 'queue=True'])
@@ -276,15 +341,17 @@ def main():
       args.remove(option)
 
   if len(args) == 0:
-    showUsage(None)
+    showUsage(options, None)
 
   commands = {
     "help": showUsage,
+    "listhostgroups": listhostgroups,
     "includedhosts": includedhosts,
     "excludedhosts": excludedhosts,
     "includehost": includehost,
     "excludehost": excludehost,
     "removehost": removehost,
+    "listportgroups": listportgroups,
     "listports": listports,
     "addport": addport,
     "removeport": removeport,
@@ -292,12 +359,30 @@ def main():
     "addportgroup": addportgroup
   }
 
+  code=1
+
+  try:
+    lockAttempts = 0
+    maxAttempts = 30
+    while lockAttempts < maxAttempts:
+      lockAttempts = lockAttempts + 1
+      try:
+        f = open(lockFile, "x")
+        f.close()
+        break
+      except:
+        time.sleep(2)
+
+    if lockAttempts == maxAttempts:
+      print("Lock file (" + lockFile + ") could not be created; proceeding without lock.")
+
     cmd = commands.get(args[0], showUsage)
-  code = cmd(args[1:])
-
-
-  if code == 0 and "--apply" in options:
-    code = apply()
+    code = cmd(options, args[1:])
+  finally:
+    try:
+      os.remove(lockFile)
+    except:
+      print("Lock file (" + lockFile + ") already removed")
 
   sys.exit(code)
 

salt/common/tools/sbin/so-fleet-restart

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

salt/common/tools/sbin/so-fleet-setup

@@ -16,7 +16,7 @@ if [ ! "$(docker ps -q -f name=so-fleet)" ]; then
 fi
 
 docker exec so-fleet fleetctl config set --address https://127.0.0.1:8080 --tls-skip-verify --url-prefix /fleet
-docker exec -it so-fleet bash -c 'while [[ "$(curl -s -o /dev/null --insecure -w ''%{http_code}'' https://127.0.0.1:8080/fleet)" != "301" ]]; do sleep 5; done'
+docker exec so-fleet bash -c 'while [[ "$(curl -s -o /dev/null --insecure -w ''%{http_code}'' https://127.0.0.1:8080/fleet)" != "301" ]]; do sleep 5; done'
 docker exec so-fleet fleetctl setup --email $1 --password $2
 
 docker exec so-fleet fleetctl apply -f /packs/palantir/Fleet/Endpoints/MacOS/osquery.yaml

salt/common/tools/sbin/so-fleet-start

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

salt/common/tools/sbin/so-fleet-stop

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

salt/common/tools/sbin/so-fleet-user-add

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

salt/common/tools/sbin/so-fleet-user-enable

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

salt/common/tools/sbin/so-grafana-restart

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

salt/common/tools/sbin/so-grafana-start

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

salt/common/tools/sbin/so-grafana-stop

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

salt/common/tools/sbin/so-idstools-restart

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

salt/common/tools/sbin/so-idstools-start

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

salt/common/tools/sbin/so-idstools-stop

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

salt/common/tools/sbin/so-image-common

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -16,7 +16,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 # NOTE: This script depends on so-common
-IMAGEREPO=securityonion
+IMAGEREPO=security-onion-solutions
 
 container_list() {
   MANAGERCHECK=$1
@@ -30,6 +30,7 @@ container_list() {
 
   if [ $MANAGERCHECK == 'so-import' ]; then
     TRUSTED_CONTAINERS=(
+      "so-acng"
       "so-elasticsearch"
       "so-filebeat"
       "so-idstools"
@@ -46,20 +47,17 @@ container_list() {
     TRUSTED_CONTAINERS=(
       "so-acng"
       "so-curator"
-      "so-domainstats"
       "so-elastalert"
       "so-elasticsearch"
       "so-filebeat"
       "so-fleet"
       "so-fleet-launcher"
-      "so-freqserver"
       "so-grafana"
       "so-idstools"
       "so-influxdb"
       "so-kibana"
       "so-kratos"
       "so-logstash"
-      "so-minio"
       "so-mysql"
       "so-nginx"
       "so-pcaptools"
@@ -103,7 +101,7 @@ update_docker_containers() {
   local PROGRESS_CALLBACK=$3
   local LOG_FILE=$4
 
-  local CONTAINER_REGISTRY=quay.io
+  local CONTAINER_REGISTRY=ghcr.io
   local SIGNPATH=/root/sosigs
   
   if [ -z "$CURLTYPE" ]; then
@@ -126,12 +124,19 @@ update_docker_containers() {
     container_list
   fi
 
-  # Let's make sure we have the public key
-  curl -sSL https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS | gpg --import -  >> "$LOG_FILE" 2>&1
-  
   rm -rf $SIGNPATH >> "$LOG_FILE" 2>&1 
   mkdir -p $SIGNPATH >> "$LOG_FILE" 2>&1 
 
+  # Let's make sure we have the public key
+  retry 50 10 "curl -sSL https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS -o $SIGNPATH/KEYS" >> "$LOG_FILE" 2>&1
+  result=$?
+  if [[ $result -eq 0 ]]; then
+    cat $SIGNPATH/KEYS | gpg --import - >> "$LOG_FILE" 2>&1
+  else
+    echo "Failed to pull signature key file: $result"
+    exit 1
+  fi
+  
   # Download the containers from the interwebs
   for i in "${TRUSTED_CONTAINERS[@]}"
   do
@@ -143,10 +148,10 @@ update_docker_containers() {
 
     # Pull down the trusted docker image
     local image=$i:$VERSION$IMAGE_TAG_SUFFIX
-    docker pull $CONTAINER_REGISTRY/$IMAGEREPO/$image >> "$LOG_FILE" 2>&1 
+    retry 50 10 "docker pull $CONTAINER_REGISTRY/$IMAGEREPO/$image" >> "$LOG_FILE" 2>&1 
     
     # Get signature
-    curl -A "$CURLTYPE/$CURRENTVERSION/$OS/$(uname -r)" https://sigs.securityonion.net/$VERSION/$i:$VERSION$IMAGE_TAG_SUFFIX.sig --output $SIGNPATH/$image.sig >> "$LOG_FILE" 2>&1 
+    retry 50 10 "curl -A '$CURLTYPE/$CURRENTVERSION/$OS/$(uname -r)' https://sigs.securityonion.net/$VERSION/$i:$VERSION$IMAGE_TAG_SUFFIX.sig --output $SIGNPATH/$image.sig" >> "$LOG_FILE" 2>&1
     if [[ $? -ne 0 ]]; then
       echo "Unable to pull signature file for $image" >> "$LOG_FILE" 2>&1 
       exit 1

salt/common/tools/sbin/so-import-pcap

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

salt/common/tools/sbin/so-index-list

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -15,8 +15,4 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-{% if grains['role'] in ['so-node','so-heavynode'] %}
-curl -X GET -k -L https://localhost:9200/_cat/indices?v
-{% else %}
-curl -X GET -L localhost:9200/_cat/indices?v
-{% endif %}
+curl -X GET -k -L "https://localhost:9200/_cat/indices?v&s=index"

salt/common/tools/sbin/so-influxdb-restart

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

salt/common/tools/sbin/so-influxdb-start

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

salt/common/tools/sbin/so-influxdb-stop

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

salt/common/tools/sbin/so-kibana-config-export

@@ -5,7 +5,7 @@
 # {%- set FLEET_IP = salt['pillar.get']('global:fleet_ip', '') %}
 # {%- set MANAGER = salt['pillar.get']('global:url_base', '') %}
 #
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

salt/common/tools/sbin/so-kibana-restart

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

salt/common/tools/sbin/so-kibana-space-defaults

@@ -0,0 +1,13 @@
+. /usr/sbin/so-common
+
+wait_for_web_response "http://localhost:5601/app/kibana" "Elastic"
+## This hackery will be removed if using Elastic Auth ##
+
+# Let's snag a cookie from Kibana
+THECOOKIE=$(curl -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}')
+
+# Disable certain Features from showing up in the Kibana UI
+echo
+echo "Setting up default Space:"
+curl -b "sid=$THECOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["ml","enterpriseSearch","siem","logs","infrastructure","apm","uptime","monitoring","stackAlerts","actions","fleet"]} ' >> /opt/so/log/kibana/misc.log
+echo

salt/common/tools/sbin/so-kibana-start

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

salt/common/tools/sbin/so-kibana-stop

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

salt/common/tools/sbin/so-logstash-events

@@ -0,0 +1,25 @@
+#!/bin/bash
+#
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>
+{%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%}
+
+. /usr/sbin/so-common
+
+if [ "$1" == "" ]; then
+        for i in $(curl -s -L http://{{ NODEIP }}:9600/_node/stats | jq .pipelines | jq '. | to_entries | .[].key' | sed 's/\"//g'); do echo ${i^}:; curl -s localhost:9600/_node/stats | jq .pipelines.$i.events; done
+else
+        curl -s -L http://{{ NODEIP }}:9600/_node/stats | jq .pipelines.$1.events
+fi

salt/common/tools/sbin/so-logstash-get-parsed

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

salt/common/tools/sbin/so-logstash-get-unparsed

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

salt/common/tools/sbin/so-logstash-pipeline-stats

@@ -0,0 +1,25 @@
+#!/bin/bash
+#
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>
+{%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%}
+
+. /usr/sbin/so-common
+
+if [ "$1" == "" ]; then
+        curl -s -L http://{{ NODEIP }}:9600/_node/stats | jq .pipelines 
+else
+        curl -s -L http://{{ NODEIP }}:9600/_node/stats | jq .pipelines.$1
+fi

salt/common/tools/sbin/so-logstash-restart

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

salt/common/tools/sbin/so-logstash-start

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

salt/common/tools/sbin/so-logstash-stop

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

salt/common/tools/sbin/so-monitor-add

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+. /usr/sbin/so-common
+
+usage() {
+  read -r -d '' message <<- EOM
+		usage: so-monitor-add [-h] NIC
+
+    positional arguments:
+      NIC           The interface to add to the monitor bond (ex: eth2)
+
+    optional arguments:
+      -h, --help    Show this help message and exit
+	EOM
+  echo "$message"
+  exit 1
+}
+
+if [[ $# -eq 0 || $# -gt 1 ]] || [[ $1 == '-h' || $1 == '--help' ]]; then
+  usage
+fi
+
+add_interface_bond0 "$1"

salt/common/tools/sbin/so-mysql-restart

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

salt/common/tools/sbin/so-mysql-start

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

salt/common/tools/sbin/so-mysql-stop

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

salt/common/tools/sbin/so-nginx-restart

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

salt/common/tools/sbin/so-nginx-start

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

salt/common/tools/sbin/so-nginx-stop

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by