From 79c7af9a31c50058dc13e089654c33419f0d27a5 Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Mon, 15 Mar 2021 10:48:24 -0400
Subject: [PATCH 01/27] soup for 2.3.40

---
 salt/common/tools/sbin/soup | 60 ++++++++++++++++++++++++-------------
 1 file changed, 39 insertions(+), 21 deletions(-)

diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup
index 264f1b603..996f0354f 100755
--- a/salt/common/tools/sbin/soup
+++ b/salt/common/tools/sbin/soup
@@ -19,6 +19,7 @@
 
 UPDATE_DIR=/tmp/sogh/securityonion
 INSTALLEDVERSION=$(cat /etc/soversion)
+POSTVERSION=$INSTALLEDVERSION
 INSTALLEDSALTVERSION=$(salt --versions-report | grep Salt: | awk {'print $2'})
 DEFAULT_SALT_DIR=/opt/so/saltstack/default
 BATCHSIZE=5
@@ -243,18 +244,6 @@ masterunlock() {
   fi
 }
 
-playbook() {
-  echo "Applying playbook settings"
-  if [[ "$INSTALLEDVERSION" =~ rc.1 ]]; then
-    salt-call state.apply playbook.OLD_db_init
-    rm -f /opt/so/rules/elastalert/playbook/*.yaml
-    so-playbook-ruleupdate >> /root/soup_playbook_rule_update.log 2>&1 &
-  fi
-  if [[ "$INSTALLEDVERSION" != 2.3.30 ]]; then
-    so-playbook-sigma-refresh >> /root/soup_playbook_sigma_refresh.log 2>&1 &
-  fi
-}
-
 pillar_changes() {
     # This function is to add any new pillar items if needed.
     echo "Checking to see if pillar changes are needed."
@@ -266,6 +255,26 @@ pillar_changes() {
     [[ "$INSTALLEDVERSION" == 2.3.20 || "$INSTALLEDVERSION" == 2.3.21 ]] && up_2.3.2X_to_2.3.30
 }
 
+post_changes() {
+    # This function is to add any new pillar items if needed.
+    echo "Checking to see if pillar changes are needed."
+    
+    [[ "$POSTVERSION" =~ rc.1 ]] && post_rc1_to_rc2
+    [[ "$POSTVERSION" == 2.3.20 || "$POSTVERSION" == 2.3.21 ]] && post_2.3.2X_to_2.3.30
+}
+
+post_rc1_to_2.3.21() {
+  salt-call state.apply playbook.OLD_db_init
+  rm -f /opt/so/rules/elastalert/playbook/*.yaml
+  so-playbook-ruleupdate >> /root/soup_playbook_rule_update.log 2>&1 &
+  POSTVERSION=2.3.21
+}
+
+post_2.3.2X_to_2.3.30() {
+  so-playbook-sigma-refresh >> /root/soup_playbook_sigma_refresh.log 2>&1 &
+  POSTVERSION=2.3.30
+}
+
 rc1_to_rc2() {
 
   # Move the static file to global.sls
@@ -410,16 +419,25 @@ up_2.3.2X_to_2.3.30() {
   check_log_size_limit
 }
 
-space_check() {
-  # Check to see if there is enough space
+verify_upgradespace() {
   CURRENTSPACE=$(df -BG / | grep -v Avail | awk '{print $4}' | sed 's/.$//')
   if [ "$CURRENTSPACE" -lt "10" ]; then
-      echo "You are low on disk space. Upgrade will try and clean up space.";
-      clean_dockers
+      echo "You are low on disk space.";
+      return 1
   else
-      echo "Plenty of space for upgrading"
-  fi
-  
+      return 0
+}
+
+upgrade_space() {
+  if ! verify_upgradespace; then
+    clean_dockers
+    if ! verify_upgradespace; then
+      echo "There is not enough space to perform the upgrade. Please free up space and try again"
+      exit 1
+    fi
+  else
+      echo "You have enough space for upgrade. Proceeding with soup."
+  fi  
 }
 
 thehive_maint() {
@@ -599,7 +617,7 @@ fi
 
 echo "Let's see if we need to update Security Onion."
 upgrade_check
-space_check
+upgrade_space
 
 echo "Checking for Salt Master and Minion updates."
 upgrade_check_salt
@@ -704,7 +722,7 @@ echo "Starting Salt Master service."
 systemctl start salt-master
 echo "Running a highstate. This could take several minutes."
 salt-call state.highstate -l info queue=True
-playbook
+post_changes
 unmount_update
 thehive_maint
 

From ab75d0e5634ad13582a006f14a6ff0dad5a556bb Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Mon, 15 Mar 2021 10:51:31 -0400
Subject: [PATCH 02/27] soup for 2.3.40

---
 salt/common/tools/sbin/soup | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup
index 996f0354f..c70726210 100755
--- a/salt/common/tools/sbin/soup
+++ b/salt/common/tools/sbin/soup
@@ -244,10 +244,10 @@ masterunlock() {
   fi
 }
 
-pillar_changes() {
+preupgrade_changes() {
     # This function is to add any new pillar items if needed.
-    echo "Checking to see if pillar changes are needed."
-    
+    echo "Checking to see if changes are needed."
+
     [[ "$INSTALLEDVERSION" =~ rc.1 ]] && rc1_to_rc2
     [[ "$INSTALLEDVERSION" =~ rc.2 ]] && rc2_to_rc3
     [[ "$INSTALLEDVERSION" =~ rc.3 ]] && rc3_to_2.3.0
@@ -255,9 +255,9 @@ pillar_changes() {
     [[ "$INSTALLEDVERSION" == 2.3.20 || "$INSTALLEDVERSION" == 2.3.21 ]] && up_2.3.2X_to_2.3.30
 }
 
-post_changes() {
+postupgrade_changes() {
     # This function is to add any new pillar items if needed.
-    echo "Checking to see if pillar changes are needed."
+    echo "Running post upgrade processes."
     
     [[ "$POSTVERSION" =~ rc.1 ]] && post_rc1_to_rc2
     [[ "$POSTVERSION" == 2.3.20 || "$POSTVERSION" == 2.3.21 ]] && post_2.3.2X_to_2.3.30
@@ -667,8 +667,7 @@ else
   echo ""
 fi
 
-echo "Making pillar changes."
-pillar_changes
+preupgrade_changes
 echo ""
 
 if [ $is_airgap -eq 0 ]; then
@@ -722,7 +721,7 @@ echo "Starting Salt Master service."
 systemctl start salt-master
 echo "Running a highstate. This could take several minutes."
 salt-call state.highstate -l info queue=True
-post_changes
+postupgrade_changes
 unmount_update
 thehive_maint
 

From b6a785395dbae410711812ed4d4215d0d65bb410 Mon Sep 17 00:00:00 2001
From: Wes Lambert <wlambertts@gmail.com>
Date: Mon, 15 Mar 2021 15:42:13 +0000
Subject: [PATCH 03/27] Add Strelka staging directory for state

---
 salt/strelka/init.sls | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/salt/strelka/init.sls b/salt/strelka/init.sls
index 91226701d..3d916aa93 100644
--- a/salt/strelka/init.sls
+++ b/salt/strelka/init.sls
@@ -86,6 +86,13 @@ strelkaprocessed:
     - group: 939
     - makedirs: True
 
+strelkastaging:
+   file.directory:
+    - name: /nsm/strelka/staging
+    - user: 939
+    - group: 939
+    - makedirs: True
+
 strelkaunprocessed:
    file.directory:
     - name: /nsm/strelka/unprocessed
@@ -213,4 +220,4 @@ strelka_zeek_extracted_sync:
   test.fail_without_changes:
     - name: {{sls}}_state_not_allowed
 
-{% endif %}
\ No newline at end of file
+{% endif %}

From f142b754dca88f17adaaaca6a9ea2693165efe99 Mon Sep 17 00:00:00 2001
From: Wes Lambert <wlambertts@gmail.com>
Date: Mon, 15 Mar 2021 15:43:31 +0000
Subject: [PATCH 04/27] Add Strelka files.processed directory so files will be
 moved from staging to processed

---
 salt/strelka/files/filestream/filestream.yaml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/salt/strelka/files/filestream/filestream.yaml b/salt/strelka/files/filestream/filestream.yaml
index aa5d51ad1..57ef65127 100644
--- a/salt/strelka/files/filestream/filestream.yaml
+++ b/salt/strelka/files/filestream/filestream.yaml
@@ -19,7 +19,8 @@ files:
     - '/nsm/strelka/unprocessed/*'
   delete: false
   gatekeeper: true
+  processed: '/nsm/strelka/processed'
 response:
   report: 5s
 delta: 5s
-staging: '/nsm/strelka/processed'
+staging: '/nsm/strelka/staging'

From 5fe025318bf21866de33144bbac2bfa7d0b9ae6d Mon Sep 17 00:00:00 2001
From: Josh Brower <Josh@Defensivedepth.com>
Date: Mon, 15 Mar 2021 15:53:00 -0400
Subject: [PATCH 05/27] Update Sigmac mappings and config for IPs and ports

---
 .../templates/so/so-common-template.json       | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/salt/elasticsearch/templates/so/so-common-template.json b/salt/elasticsearch/templates/so/so-common-template.json
index 012c590d0..5e05896c3 100644
--- a/salt/elasticsearch/templates/so/so-common-template.json
+++ b/salt/elasticsearch/templates/so/so-common-template.json
@@ -51,7 +51,14 @@
             "match_mapping_type": "string",
             "path_match": "*.ip",
             "mapping": {
-              "type": "ip"
+              "type": "ip",
+              "fields" : {
+                "keyword" : {
+                  "ignore_above" : 45,
+                  "type" : "keyword"
+                }
+              }
+
             }
           }
        },
@@ -60,7 +67,14 @@
             "match_mapping_type": "string",
             "path_match": "*.port",
             "mapping": {
-              "type": "integer"
+              "type": "integer",
+              "fields" : {
+                "keyword" : {
+                  "ignore_above" : 10,
+                  "type" : "keyword"
+                }
+              }
+
             }
           }
        },

From 674bb342ea966569b870d419232b7bbbad2d13ee Mon Sep 17 00:00:00 2001
From: Mike Reeves <luke@geekempire.com>
Date: Mon, 15 Mar 2021 16:39:43 -0400
Subject: [PATCH 06/27] Turn off SSL Verification in Telegraf

---
 salt/telegraf/etc/telegraf.conf | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf
index b8976b8c9..f6bcbdaf5 100644
--- a/salt/telegraf/etc/telegraf.conf
+++ b/salt/telegraf/etc/telegraf.conf
@@ -618,11 +618,8 @@
 # # Read stats from one or more Elasticsearch servers or clusters
 {% if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone']   %}
  [[inputs.elasticsearch]]
-
-#   ## specify a list of one or more Elasticsearch servers
-#   # you can add username and password to your url to use basic authentication:
-#   # servers = ["http://user:pass@localhost:9200"]
    servers = ["https://{{ MANAGER }}:9200"]
+   insecure_skip_verify = true
 {% elif grains['role'] in ['so-node', 'so-hotnode', 'so-warmnode', 'so-heavynode']  %}
  [[inputs.elasticsearch]]
    servers = ["https://{{ NODEIP }}:9200"]

From 67e0d450e4c896a48d8fef869d8edd3a402ed83b Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Mon, 15 Mar 2021 17:32:36 -0400
Subject: [PATCH 07/27] Add Elastic License Prompts

---
 salt/common/tools/sbin/so-common | 58 ++++++++++++++++++++++++++++++++
 salt/common/tools/sbin/soup      | 15 ++-------
 setup/so-setup                   |  5 ++-
 3 files changed, 62 insertions(+), 16 deletions(-)

diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common
index b76ad05ff..a62e654cc 100755
--- a/salt/common/tools/sbin/so-common
+++ b/salt/common/tools/sbin/so-common
@@ -86,6 +86,19 @@ add_interface_bond0() {
 	fi
 }
 
+check_airgap() {
+  # See if this is an airgap install
+  AIRGAP=$(cat /opt/so/saltstack/local/pillar/global.sls | grep airgap: | awk '{print $2}')
+  if [[ "$AIRGAP" == "True" ]]; then
+      is_airgap=0
+      UPDATE_DIR=/tmp/soagupdate/SecurityOnion
+      AGDOCKER=/tmp/soagupdate/docker
+      AGREPO=/tmp/soagupdate/Packages
+  else 
+      is_airgap=1
+  fi
+}
+
 check_container() {
 	docker ps | grep "$1:" > /dev/null 2>&1
 	return $?
@@ -97,6 +110,45 @@ check_password() {
 	return $?
 }
 
+check_elastic_license() {
+
+	[ -n "$TESTING" ] && return
+
+	# See if the user has already accepted the license
+	if [ ! -f /opt/so/state/yeselastic.txt ]
+	  elastic_license
+	else
+	  echo "Elastic License has already been accepted"
+	fi  
+}
+
+elastic_license() {
+
+read -r -d '' message <<- EOM
+\n
+Security Onion now uses the Elastic Stack binaries that fall under the Elastic license.
+
+Please review the Elastic license:
+https://raw.githubusercontent.com/elastic/elasticsearch/master/licenses/ELASTIC-LICENSE.txt
+
+Do you agree to the terms of the Elastic license?
+
+If so, type AGREE to accept the Elastic license and continue.  Otherwise, press Enter to exit this program without making any changes.
+EOM
+
+AGREED=$(whiptail --title "Security Onion Setup" --inputbox \
+"$message" 20 75)
+
+if [ "$AGREED" = 'AGREE' ]; then
+  mkdir -p /opt/so/state
+  touch /opt/so/state/yeselastic.txt 
+else
+  echo "Starting in 2.3.40 you must accept the Elastic license if you want to run Security Onion."
+  exit 1
+fi
+
+}
+
 fail() {
 	msg=$1
 	echo "ERROR: $msg"
@@ -250,6 +302,12 @@ set_minionid() {
 	MINIONID=$(lookup_grain id)
 }
 
+set_palette() {
+	if [ "$OS" == ubuntu ]; then
+	    update-alternatives --set newt-palette /etc/newt/palette.original
+    fi
+}
+
 set_version() {
 	CURRENTVERSION=0.0.0
 	if [ -f /etc/soversion ]; then
diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup
index c70726210..2a121d602 100755
--- a/salt/common/tools/sbin/soup
+++ b/salt/common/tools/sbin/soup
@@ -102,19 +102,6 @@ update_registry() {
   salt-call state.apply registry queue=True
 }
 
-check_airgap() {
-  # See if this is an airgap install
-  AIRGAP=$(cat /opt/so/saltstack/local/pillar/global.sls | grep airgap: | awk '{print $2}')
-  if [[ "$AIRGAP" == "True" ]]; then
-      is_airgap=0
-      UPDATE_DIR=/tmp/soagupdate/SecurityOnion
-      AGDOCKER=/tmp/soagupdate/docker
-      AGREPO=/tmp/soagupdate/Packages
-  else 
-      is_airgap=1
-  fi
-}
-
 check_sudoers() {
 	if grep -q "so-setup" /etc/sudoers; then
 		echo "There is an entry for so-setup in the sudoers file, this can be safely deleted using \"visudo\"."
@@ -591,6 +578,8 @@ check_airgap
 echo "Found that Security Onion $INSTALLEDVERSION is currently installed."
 echo ""
 set_os
+set_palette
+check_elastic_license
 echo ""
 if [ $is_airgap -eq 0 ]; then
   # Let's mount the ISO since this is airgap
diff --git a/setup/so-setup b/setup/so-setup
index 0af49af53..eaf7c7657 100755
--- a/setup/so-setup
+++ b/setup/so-setup
@@ -156,10 +156,8 @@ export PATH=$PATH:$local_sbin
 
 installer_prereq_packages && detect_cloud
 set_network_dev_status_list
+set_palette >> $setup_log 2>&1
 
-if [ "$OS" == ubuntu ]; then
-	update-alternatives --set newt-palette /etc/newt/palette.original >> $setup_log 2>&1
-fi
 
 # Kernel messages can overwrite whiptail screen #812
 # https://github.com/Security-Onion-Solutions/securityonion/issues/812
@@ -251,6 +249,7 @@ fi
 
 # Check if this is an airgap install
 if [[ ( $is_manager || $is_import ) && $is_iso ]]; then
+    check_elastic_license
 	whiptail_airgap
 	if [[ "$INTERWEBS" == 'AIRGAP' ]]; then
 		is_airgap=true

From 72759de97fdccc0ad0fda504d4deaced971f520f Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Mon, 15 Mar 2021 17:37:44 -0400
Subject: [PATCH 08/27] Fix so-common syntax

---
 salt/common/tools/sbin/so-common | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common
index a62e654cc..99b41e659 100755
--- a/salt/common/tools/sbin/so-common
+++ b/salt/common/tools/sbin/so-common
@@ -115,7 +115,7 @@ check_elastic_license() {
 	[ -n "$TESTING" ] && return
 
 	# See if the user has already accepted the license
-	if [ ! -f /opt/so/state/yeselastic.txt ]
+	if [ ! -f /opt/so/state/yeselastic.txt ]; then
 	  elastic_license
 	else
 	  echo "Elastic License has already been accepted"

From 12501e00796904513881980ce43e5f7fbeef66a5 Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Mon, 15 Mar 2021 17:41:45 -0400
Subject: [PATCH 09/27] Add check license to its own logic

---
 setup/so-setup | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/setup/so-setup b/setup/so-setup
index eaf7c7657..e9039f164 100755
--- a/setup/so-setup
+++ b/setup/so-setup
@@ -249,13 +249,16 @@ fi
 
 # Check if this is an airgap install
 if [[ ( $is_manager || $is_import ) && $is_iso ]]; then
-    check_elastic_license
 	whiptail_airgap
 	if [[ "$INTERWEBS" == 'AIRGAP' ]]; then
 		is_airgap=true
 	fi
 fi
 
+if [[ $is_manager || $is_import ]]; then
+	check_elastic_license
+fi
+
 if ! [[ -f $install_opt_file ]]; then
 	if [[ $is_manager && $is_sensor ]]; then
 		check_requirements "standalone"

From 9a4c4448f31e4f7efa9f20122671afc0f2d36fd8 Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Mon, 15 Mar 2021 17:45:44 -0400
Subject: [PATCH 10/27] Fix whiptail display

---
 salt/common/tools/sbin/so-common | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common
index 99b41e659..22a21de95 100755
--- a/salt/common/tools/sbin/so-common
+++ b/salt/common/tools/sbin/so-common
@@ -137,7 +137,7 @@ If so, type AGREE to accept the Elastic license and continue.  Otherwise, press
 EOM
 
 AGREED=$(whiptail --title "Security Onion Setup" --inputbox \
-"$message" 20 75)
+"$message" 20 75 3>&1 1>&2 2>&3)
 
 if [ "$AGREED" = 'AGREE' ]; then
   mkdir -p /opt/so/state

From 8d23518f9010f91d4c7beb87b401c7c9a08b4779 Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Mon, 15 Mar 2021 17:50:06 -0400
Subject: [PATCH 11/27] Update Elastic Link

---
 salt/common/tools/sbin/so-common | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common
index 22a21de95..1f8b3d54c 100755
--- a/salt/common/tools/sbin/so-common
+++ b/salt/common/tools/sbin/so-common
@@ -129,7 +129,7 @@ read -r -d '' message <<- EOM
 Security Onion now uses the Elastic Stack binaries that fall under the Elastic license.
 
 Please review the Elastic license:
-https://raw.githubusercontent.com/elastic/elasticsearch/master/licenses/ELASTIC-LICENSE.txt
+https://www.elastic.co/licensing/elastic-license
 
 Do you agree to the terms of the Elastic license?
 

From 44c75122edded7357ae11e36d76159be418baed9 Mon Sep 17 00:00:00 2001
From: Josh Brower <Josh@Defensivedepth.com>
Date: Tue, 16 Mar 2021 09:05:35 -0400
Subject: [PATCH 12/27] Update Sigmac mappings and config for IPs and ports

---
 salt/elasticsearch/files/ingest/common | 2 --
 1 file changed, 2 deletions(-)

diff --git a/salt/elasticsearch/files/ingest/common b/salt/elasticsearch/files/ingest/common
index 63eb6077a..e8e462053 100644
--- a/salt/elasticsearch/files/ingest/common
+++ b/salt/elasticsearch/files/ingest/common
@@ -32,8 +32,6 @@
     { "rename":          { "field": "category",            "target_field": "event.category",              "ignore_failure": true, "ignore_missing": true  } },
     { "rename":          { "field": "message2.community_id", "target_field": "network.community_id",  "ignore_failure": true,  "ignore_missing": true  } },
     { "lowercase":       { "field": "event.dataset", "ignore_failure": true,  "ignore_missing": true  } },
-    { "convert":         { "field": "destination.port", "type": "integer",  "ignore_failure": true,  "ignore_missing": true  } },
-    { "convert":         { "field": "source.port", "type": "integer",  "ignore_failure": true,  "ignore_missing": true  } },
     { "convert":         { "field": "log.id.uid", "type": "string",  "ignore_failure": true,  "ignore_missing": true  } },
     { "convert":         { "field": "agent.id", "type": "string",  "ignore_failure": true,  "ignore_missing": true  } },
     { "convert":         { "field": "event.severity", "type": "integer",  "ignore_failure": true,  "ignore_missing": true  } },

From 71ae5b60eac6649890a5e33e1c709379e6ef8a43 Mon Sep 17 00:00:00 2001
From: Josh Brower <Josh@Defensivedepth.com>
Date: Tue, 16 Mar 2021 09:32:40 -0400
Subject: [PATCH 13/27] Update Sigmac mappings and config for IPs and ports

---
 salt/elasticsearch/templates/so/so-common-template.json | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/salt/elasticsearch/templates/so/so-common-template.json b/salt/elasticsearch/templates/so/so-common-template.json
index 5e05896c3..ebf123fed 100644
--- a/salt/elasticsearch/templates/so/so-common-template.json
+++ b/salt/elasticsearch/templates/so/so-common-template.json
@@ -64,13 +64,12 @@
        },
        {
           "port": {
-            "match_mapping_type": "string",
             "path_match": "*.port",
             "mapping": {
               "type": "integer",
               "fields" : {
                 "keyword" : {
-                  "ignore_above" : 10,
+                  "ignore_above" : 6,
                   "type" : "keyword"
                 }
               }

From 00025e5c743c3a040b0485670d0b78f090da9ef6 Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Tue, 16 Mar 2021 09:34:53 -0400
Subject: [PATCH 14/27] Fix Syntax Error

---
 salt/common/tools/sbin/soup | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup
index 2a121d602..1a877c2e4 100755
--- a/salt/common/tools/sbin/soup
+++ b/salt/common/tools/sbin/soup
@@ -409,10 +409,11 @@ up_2.3.2X_to_2.3.30() {
 verify_upgradespace() {
   CURRENTSPACE=$(df -BG / | grep -v Avail | awk '{print $4}' | sed 's/.$//')
   if [ "$CURRENTSPACE" -lt "10" ]; then
-      echo "You are low on disk space.";
+      echo "You are low on disk space."
       return 1
   else
       return 0
+  fi
 }
 
 upgrade_space() {

From 038c58f3d51c15e6fb4d41e79155554f52b4e9bf Mon Sep 17 00:00:00 2001
From: Wes Lambert <wlambertts@gmail.com>
Date: Tue, 16 Mar 2021 14:51:16 +0000
Subject: [PATCH 15/27] Ignore TIME_WAIT when checking for Strelka frontend
 port reservation

---
 salt/strelka/init.sls | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/strelka/init.sls b/salt/strelka/init.sls
index 3d916aa93..d90484fed 100644
--- a/salt/strelka/init.sls
+++ b/salt/strelka/init.sls
@@ -103,7 +103,7 @@ strelkaunprocessed:
 # Check to see if Strelka frontend port is available
 strelkaportavailable:
     cmd.run:
-      - name: netstat -utanp | grep ":57314" | grep -qv docker && PROCESS=$(netstat -utanp | grep ":57314" | uniq) && echo "Another process ($PROCESS) appears to be using port 57314.  Please terminate this process, or reboot to ensure a clean state so that Strelka can start properly." && exit 1 || exit 0
+      - name: netstat -utanp | grep ":57314" | grep -qvE 'docker|TIME_WAIT' && PROCESS=$(netstat -utanp | grep ":57314" | uniq) && echo "Another process ($PROCESS) appears to be using port 57314.  Please terminate this process, or reboot to ensure a clean state so that Strelka can start properly." && exit 1 || exit 0
 
 strelka_coordinator:
   docker_container.running:

From 2851840e76c55a660d84ca44faff386b8c754ce4 Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Tue, 16 Mar 2021 12:18:01 -0400
Subject: [PATCH 16/27] Fix Logging

---
 salt/common/tools/sbin/soup | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup
index 1a877c2e4..701cebafc 100755
--- a/salt/common/tools/sbin/soup
+++ b/salt/common/tools/sbin/soup
@@ -25,8 +25,6 @@ DEFAULT_SALT_DIR=/opt/so/saltstack/default
 BATCHSIZE=5
 SOUP_LOG=/root/soup.log
 
-exec 3>&1 1>${SOUP_LOG} 2>&1
-
 add_common() {
   cp $UPDATE_DIR/salt/common/tools/sbin/so-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/
   cp $UPDATE_DIR/salt/common/tools/sbin/so-image-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/
@@ -554,6 +552,7 @@ verify_latest_update_script() {
 }
 
 main () {
+echo "### Preparing soup at `date` ###"
 while getopts ":b" opt; do
   case "$opt" in
     b ) # process option b
@@ -758,7 +757,7 @@ For more information, please see https://docs.securityonion.net/en/2.3/soup.html
 EOF
 
 fi
-
+echo "### soup has been served at `date` ###"
 }
 
-main "$@" | tee /dev/fd/3
+main "$@" | tee -a $SOUP_LOG

From e528d84ebee1bdd84bedf3f81db854b8261e33bc Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Tue, 16 Mar 2021 12:56:06 -0400
Subject: [PATCH 17/27] Update Elastic License Text

---
 salt/common/tools/sbin/so-common | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common
index 1f8b3d54c..4a934d099 100755
--- a/salt/common/tools/sbin/so-common
+++ b/salt/common/tools/sbin/so-common
@@ -126,14 +126,15 @@ elastic_license() {
 
 read -r -d '' message <<- EOM
 \n
-Security Onion now uses the Elastic Stack binaries that fall under the Elastic license.
+Starting in Elastic Stack version 7.11, the Elastic Stack binaries are only available under the Elastic License:
+https://blog.securityonion.net/2021/02/elastic-license-changes-and-security.html
 
-Please review the Elastic license:
+Please review the Elastic License:
 https://www.elastic.co/licensing/elastic-license
 
-Do you agree to the terms of the Elastic license?
+Do you agree to the terms of the Elastic License?
 
-If so, type AGREE to accept the Elastic license and continue.  Otherwise, press Enter to exit this program without making any changes.
+If so, type AGREE to accept the Elastic License and continue.  Otherwise, press Enter to exit this program without making any changes.
 EOM
 
 AGREED=$(whiptail --title "Security Onion Setup" --inputbox \

From 7a02150389f03fb63283f1912dcac7c3aa417194 Mon Sep 17 00:00:00 2001
From: Wes Lambert <wlambertts@gmail.com>
Date: Tue, 16 Mar 2021 17:16:55 +0000
Subject: [PATCH 18/27] Reverse Zeek index close/delete count for Curator

---
 setup/so-functions | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/setup/so-functions b/setup/so-functions
index c48f08819..6a0874756 100755
--- a/setup/so-functions
+++ b/setup/so-functions
@@ -1642,8 +1642,8 @@ manager_global() {
 		"    so-zeek:"\
 		"      shards: 5"\
 		"      warm: 7"\
-		"      close: 365"\
-		"      delete: 45"\
+		"      close: 45"\
+		"      delete: 365"\
 		"minio:"\
 		"  access_key: '$ACCESS_KEY'"\
 		"  access_secret: '$ACCESS_SECRET'"\

From c855e0a55ae9a3fa29e352f7248d82211813dd72 Mon Sep 17 00:00:00 2001
From: Josh Brower <Josh@Defensivedepth.com>
Date: Tue, 16 Mar 2021 13:48:13 -0400
Subject: [PATCH 19/27] Disable certain Features within the default space

---
 salt/kibana/bin/so-kibana-config-load | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/salt/kibana/bin/so-kibana-config-load b/salt/kibana/bin/so-kibana-config-load
index c857b2857..30c21e8d6 100644
--- a/salt/kibana/bin/so-kibana-config-load
+++ b/salt/kibana/bin/so-kibana-config-load
@@ -24,3 +24,6 @@ THECOOKIE=$(curl -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}
 
 # Load saved objects
 curl -b "sid=$THECOOKIE" -L -X POST "localhost:5601/api/saved_objects/_import?overwrite=true" -H "kbn-xsrf: true" --form file=@/opt/so/conf/kibana/saved_objects.ndjson > /dev/null 2>&1
+
+# Disable certain Features from showing up in the Kibana UI
+curl -b "sid=$THECOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["enterpriseSearch","siem","logs","infrastructure","apm","uptime","monitoring","stackAlerts","actions","fleet"]} '
\ No newline at end of file

From abd4f920882c614e0e98f867b2500d27ab8356dc Mon Sep 17 00:00:00 2001
From: Josh Brower <Josh@Defensivedepth.com>
Date: Tue, 16 Mar 2021 13:53:28 -0400
Subject: [PATCH 20/27] Cleanup curl output

---
 salt/kibana/bin/so-kibana-config-load | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/salt/kibana/bin/so-kibana-config-load b/salt/kibana/bin/so-kibana-config-load
index 30c21e8d6..0ee2508eb 100644
--- a/salt/kibana/bin/so-kibana-config-load
+++ b/salt/kibana/bin/so-kibana-config-load
@@ -26,4 +26,7 @@ THECOOKIE=$(curl -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}
 curl -b "sid=$THECOOKIE" -L -X POST "localhost:5601/api/saved_objects/_import?overwrite=true" -H "kbn-xsrf: true" --form file=@/opt/so/conf/kibana/saved_objects.ndjson > /dev/null 2>&1
 
 # Disable certain Features from showing up in the Kibana UI
-curl -b "sid=$THECOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["enterpriseSearch","siem","logs","infrastructure","apm","uptime","monitoring","stackAlerts","actions","fleet"]} '
\ No newline at end of file
+
+echo
+echo "Setting up default Space:"
+curl -b "sid=$THECOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["enterpriseSearch","siem","logs","infrastructure","apm","uptime","monitoring","stackAlerts","actions","fleet"]} ' | jq
\ No newline at end of file

From 38a497932c033886657f37398392c74232af53c0 Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Tue, 16 Mar 2021 16:36:35 -0400
Subject: [PATCH 21/27] 
 https://github.com/Security-Onion-Solutions/securityonion/issues/3288

---
 salt/firewall/init.sls | 41 +++++++++++++++++++++--------------------
 1 file changed, 21 insertions(+), 20 deletions(-)

diff --git a/salt/firewall/init.sls b/salt/firewall/init.sls
index c9618554d..3d4d5ef6b 100644
--- a/salt/firewall/init.sls
+++ b/salt/firewall/init.sls
@@ -26,15 +26,6 @@ iptables_fix_fwd:
     - position: 1
     - target: DOCKER-USER
 
-# Allow related/established sessions
-iptables_allow_established:
-  iptables.append:
-    - table: filter
-    - chain: INPUT
-    - jump: ACCEPT
-    - match: conntrack
-    - ctstate: 'RELATED,ESTABLISHED'
-
 # I like pings
 iptables_allow_pings:
   iptables.append:
@@ -77,17 +68,6 @@ enable_docker_user_fw_policy:
     - out-interface: docker0
     - position: 1
 
-enable_docker_user_established:
-  iptables.insert:
-    - table: filter
-    - chain: DOCKER-USER
-    - jump: ACCEPT
-    - in-interface: '!docker0'
-    - out-interface: docker0
-    - position: 1
-    - match: conntrack
-    - ctstate: 'RELATED,ESTABLISHED'
-
 {% set count = namespace(value=0) %}
 {% for chain, hg in assigned_hostgroups.chain.items() %}
   {% for hostgroup, portgroups in assigned_hostgroups.chain[chain].hostgroups.items() %}
@@ -120,6 +100,27 @@ enable_docker_user_established:
   {% endfor %}
 {% endfor %}
 
+# Allow related/established sessions
+iptables_allow_established:
+  iptables.append:
+    - table: filter
+    - chain: INPUT
+    - jump: ACCEPT
+    - position: 1
+    - match: conntrack
+    - ctstate: 'RELATED,ESTABLISHED'
+
+enable_docker_user_established:
+  iptables.insert:
+    - table: filter
+    - chain: DOCKER-USER
+    - jump: ACCEPT
+    - in-interface: '!docker0'
+    - out-interface: docker0
+    - position: 1
+    - match: conntrack
+    - ctstate: 'RELATED,ESTABLISHED'
+
 # Block icmp timestamp response
 block_icmp_timestamp_reply:
   iptables.append:

From 388524ec4edaa801a16cab513052e00297fec384 Mon Sep 17 00:00:00 2001
From: Doug Burks <doug.burks@gmail.com>
Date: Tue, 16 Mar 2021 16:39:14 -0400
Subject: [PATCH 22/27] Shorten Elastic License URL to avoid line wrap

---
 salt/common/tools/sbin/so-common | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common
index 4a934d099..1f63f0073 100755
--- a/salt/common/tools/sbin/so-common
+++ b/salt/common/tools/sbin/so-common
@@ -127,7 +127,7 @@ elastic_license() {
 read -r -d '' message <<- EOM
 \n
 Starting in Elastic Stack version 7.11, the Elastic Stack binaries are only available under the Elastic License:
-https://blog.securityonion.net/2021/02/elastic-license-changes-and-security.html
+https://securityonion.net/elastic-license
 
 Please review the Elastic License:
 https://www.elastic.co/licensing/elastic-license

From 7719a26a96f2ca0d51519235fd778a05607da8d6 Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Tue, 16 Mar 2021 18:53:41 -0400
Subject: [PATCH 23/27] Change ES Heap calculation

---
 setup/so-functions | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/setup/so-functions b/setup/so-functions
index 6a0874756..f53b696b1 100755
--- a/setup/so-functions
+++ b/setup/so-functions
@@ -1201,8 +1201,13 @@ es_heapsize() {
 		# https://www.elastic.co/guide/en/elasticsearch/guide/current/heap-sizing.html
 		ES_HEAP_SIZE="25000m"
 	else
-		# Set heap size to 25% of available memory
-		ES_HEAP_SIZE=$(( total_mem / 4 ))"m"
+		# Set heap size to 33% of available memory
+		ES_HEAP_SIZE=$(( total_mem / 3 ))"m"
+		if [ "$ES_HEAP_SIZE" -ge 25001 ] ; then
+		    ES_HEAP_SIZE="25000m"
+		else
+		    ES_HEAP_SIZE=$ES_HEAP_SIZE
+		fi
 	fi
 	export ES_HEAP_SIZE
 

From 137e1a699d08bd36653b4bd379438b7f7ce7610c Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Tue, 16 Mar 2021 19:01:10 -0400
Subject: [PATCH 24/27] Fix the math

---
 setup/so-functions | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/setup/so-functions b/setup/so-functions
index f53b696b1..06f845737 100755
--- a/setup/so-functions
+++ b/setup/so-functions
@@ -1202,11 +1202,11 @@ es_heapsize() {
 		ES_HEAP_SIZE="25000m"
 	else
 		# Set heap size to 33% of available memory
-		ES_HEAP_SIZE=$(( total_mem / 3 ))"m"
+		ES_HEAP_SIZE=$(( total_mem / 3 ))
 		if [ "$ES_HEAP_SIZE" -ge 25001 ] ; then
 		    ES_HEAP_SIZE="25000m"
 		else
-		    ES_HEAP_SIZE=$ES_HEAP_SIZE
+		    ES_HEAP_SIZE=$ES_HEAP_SIZE"m"
 		fi
 	fi
 	export ES_HEAP_SIZE

From bba9913be1267a7037d3de0b848f08c3e3566696 Mon Sep 17 00:00:00 2001
From: Josh Brower <Josh@Defensivedepth.com>
Date: Wed, 17 Mar 2021 08:25:25 -0400
Subject: [PATCH 25/27] Temp revert Fleet Live Query to Hunt

---
 pillar/logstash/manager.sls | 1 -
 pillar/logstash/search.sls  | 3 +--
 2 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/pillar/logstash/manager.sls b/pillar/logstash/manager.sls
index 1aa445c97..6f3ba495b 100644
--- a/pillar/logstash/manager.sls
+++ b/pillar/logstash/manager.sls
@@ -3,7 +3,6 @@ logstash:
   pipelines:
     manager:
       config:
-        - so/0008_input_fleet_livequery.conf.jinja
         - so/0009_input_beats.conf      
         - so/0010_input_hhbeats.conf
         - so/9999_output_redis.conf.jinja
diff --git a/pillar/logstash/search.sls b/pillar/logstash/search.sls
index 8ae84fe1f..2da8e6c59 100644
--- a/pillar/logstash/search.sls
+++ b/pillar/logstash/search.sls
@@ -7,8 +7,7 @@ logstash:
         - so/9000_output_zeek.conf.jinja
         - so/9002_output_import.conf.jinja
         - so/9034_output_syslog.conf.jinja
-        - so/9100_output_osquery.conf.jinja
-        - so/9101_output_osquery_livequery.conf.jinja        
+        - so/9100_output_osquery.conf.jinja  
         - so/9400_output_suricata.conf.jinja
         - so/9500_output_beats.conf.jinja
         - so/9600_output_ossec.conf.jinja

From d32c1de411a0eca0a0738bc34683d0ba219d987f Mon Sep 17 00:00:00 2001
From: Doug Burks <doug.burks@gmail.com>
Date: Wed, 17 Mar 2021 09:00:46 -0400
Subject: [PATCH 26/27] FEATURE: soup should provide some initial information
 and then prompt the user to continue #3486

---
 salt/common/tools/sbin/soup | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup
index 701cebafc..75e924116 100755
--- a/salt/common/tools/sbin/soup
+++ b/salt/common/tools/sbin/soup
@@ -747,6 +747,7 @@ NUM_MINIONS=$(ls /opt/so/saltstack/local/pillar/minions/*_*.sls | wc -l)
 if [ $NUM_MINIONS -gt 1 ]; then
 
 	cat << EOF
+	
 This appears to be a distributed deployment. Other nodes should update themselves at the next Salt highstate (typically within 15 minutes). Do not manually restart anything until you know that all the search/heavy nodes in your deployment are updated. This is especially important if you are using true clustering for Elasticsearch.
 
 Each minion is on a random 15 minute check-in period and things like network bandwidth can be a factor in how long the actual upgrade takes. If you have a heavy node on a slow link, it is going to take a while to get the containers to it. Depending on what changes happened between the versions, Elasticsearch might not be able to talk to said heavy node until the update is complete.
@@ -760,4 +761,20 @@ fi
 echo "### soup has been served at `date` ###"
 }
 
+cat << EOF
+
+SOUP - Security Onion UPdater
+
+Please review the following for more information about the update process and recent updates:
+https://docs.securityonion.net/soup
+https://blog.securityonion.net
+
+Please note that soup only updates Security Onion components and does NOT update the underlying operating system (OS). When you installed Security Onion, there was an option to automatically update the OS packages. If you did not enable this option, then you will want to ensure that the OS is fully updated before running soup.
+
+Press Enter to continue or Ctrl-C to cancel.
+
+EOF
+
+read input
+
 main "$@" | tee -a $SOUP_LOG

From 68ce7a902da467d80ad0edb088b8450822956785 Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Wed, 17 Mar 2021 09:14:19 -0400
Subject: [PATCH 27/27] insert instead of append

---
 salt/firewall/init.sls | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/firewall/init.sls b/salt/firewall/init.sls
index 3d4d5ef6b..1d6ba350f 100644
--- a/salt/firewall/init.sls
+++ b/salt/firewall/init.sls
@@ -102,7 +102,7 @@ enable_docker_user_fw_policy:
 
 # Allow related/established sessions
 iptables_allow_established:
-  iptables.append:
+  iptables.insert:
     - table: filter
     - chain: INPUT
     - jump: ACCEPT