From b571eeb8e6425b197b0b2245d3dfb367b1bb5f93 Mon Sep 17 00:00:00 2001
From: DefensiveDepth <Josh@defensivedepth.com>
Date: Wed, 27 Mar 2024 14:58:16 -0400
Subject: [PATCH 01/62] Initial cut of .70 soup changes

---
 salt/manager/tools/sbin/soup | 74 ++++++++++++++++++++++++++++++++++++
 1 file changed, 74 insertions(+)

diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup
index a585f877c..87d88a57e 100755
--- a/salt/manager/tools/sbin/soup
+++ b/salt/manager/tools/sbin/soup
@@ -357,6 +357,7 @@ preupgrade_changes() {
     [[ "$INSTALLEDVERSION" == 2.4.30 ]] && up_to_2.4.40
     [[ "$INSTALLEDVERSION" == 2.4.40 ]] && up_to_2.4.50
     [[ "$INSTALLEDVERSION" == 2.4.50 ]] && up_to_2.4.60
+    [[ "$INSTALLEDVERSION" == 2.4.60 ]] && up_to_2.4.70
     true
 }
 
@@ -373,6 +374,7 @@ postupgrade_changes() {
     [[ "$POSTVERSION" == 2.4.30 ]] && post_to_2.4.40
     [[ "$POSTVERSION"  == 2.4.40 ]] && post_to_2.4.50
     [[ "$POSTVERSION"  == 2.4.50 ]] && post_to_2.4.60
+    [[ "$POSTVERSION"  == 2.4.60 ]] && post_to_2.4.70
     true
 }
 
@@ -435,6 +437,11 @@ post_to_2.4.60() {
   POSTVERSION=2.4.60
 }
 
+post_to_2.4.70() {
+  echo "Nothing to apply"
+  POSTVERSION=2.4.70
+}
+
 repo_sync() {
   echo "Sync the local repo."
   su socore -c '/usr/sbin/so-repo-sync' || fail "Unable to complete so-repo-sync."
@@ -574,6 +581,73 @@ up_to_2.4.60() {
   INSTALLEDVERSION=2.4.60
 }
 
+up_to_2.4.70() {
+  # Start SOC Detections migration
+  mkdir -p /nsm/backup/detections-migration/{suricata,sigma/rules,elastalert}
+
+  # Remove cronjobs
+  crontab -l | grep -v 'so-playbook-sync_cron' | crontab -
+  crontab -l | grep -v 'so-playbook-ruleupdate_cron' | crontab -
+
+  # Check for active Elastalert rules
+  active_rules_count=$(find /opt/so/rules/elastalert/playbook/ -type f -name "*.yaml" | wc -l)
+
+  if [[ "$active_rules_count" -gt 0 ]]; then
+      # Prompt the user to AGREE if active Elastalert rules found
+      echo
+      echo "$active_rules_count Active Elastalert/Playbook rules found."
+      echo "In preparation for the new Detections module, they will be backed up and then disabled."
+      echo
+      echo "If you would like to proceed, then type AGREE and press ENTER."
+      echo
+      # Read user input
+      read INPUT
+      if [ "${INPUT^^}" != 'AGREE' ]; then exit 0; fi
+
+      echo "Backing up the Elastalert rules..."
+      rsync -av --stats /opt/so/rules/elastalert/playbook/*.yaml /nsm/backup/detections-migration/elastalert/
+
+      # Verify that rsync completed successfully
+      if [[ $? -eq 0 ]]; then
+          # Delete the Elastlaert rules
+          rm -f /opt/so/rules/elastalert/playbook/*.yaml
+          echo "Active Elastalert rules have been backed up."
+      else
+          echo "Error: rsync failed to copy the files. Active Elastalert rules have not been backed up."
+          exit 1
+      fi
+  fi
+
+  echo
+  echo "Exporting Sigma rules from Playbook..."
+  MYSQLPW=$(lookup_pillar_secret mysql)
+
+  docker exec so-mysql sh -c "exec mysql -uroot -p${MYSQLPW} -D playbook -sN -e \"SELECT id, value FROM custom_values WHERE value LIKE '%View Sigma%'\"" | while read -r id value; do
+      echo -e "$value" > "/nsm/backup/detections-migration/sigma/rules/$id.yaml"
+  done
+
+  echo
+  echo "Exporting Sigma Filters from Playbook..."
+  docker exec so-mysql sh -c "exec mysql -uroot -p${MYSQLPW} -D playbook -sN -e \"SELECT issues.subject as title, custom_values.value as filter FROM issues JOIN custom_values ON issues.id = custom_values.customized_id WHERE custom_values.value LIKE '%sofilter%'\"" > /nsm/backup/detections-migration/sigma/custom-filters.txt
+
+  echo
+  echo "Backing up Playbook database..."
+  docker exec so-mysql sh -c "mysqldump -uroot -p${MYSQLPW} --databases playbook > /tmp/playbook-dump"
+  docker cp so-mysql:/tmp/playbook-dump /nsm/backup/detections-migration/sigma/playbook-dump.sql
+
+  echo
+  echo "Stopping Playbook services..."
+  so-playbook-stop
+  so-mysql-stop
+  so-soctopus-stop
+
+  # What about cleaning up various so-utilities like so-playbook-restart?
+  echo
+  echo "Playbook Migration is complete...."
+
+  INSTALLEDVERSION=2.4.70
+}
+
 determine_elastic_agent_upgrade() {
   if [[ $is_airgap -eq 0 ]]; then
     update_elastic_agent_airgap

From ba262ee01a9fada97e21fe82019598ced83529fa Mon Sep 17 00:00:00 2001
From: DefensiveDepth <Josh@defensivedepth.com>
Date: Wed, 27 Mar 2024 15:43:25 -0400
Subject: [PATCH 02/62] Check to see if Playbook is enabled

---
 salt/manager/tools/sbin/soup | 85 +++++++++++++++++++-----------------
 1 file changed, 44 insertions(+), 41 deletions(-)

diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup
index 87d88a57e..62a579e18 100755
--- a/salt/manager/tools/sbin/soup
+++ b/salt/manager/tools/sbin/soup
@@ -589,52 +589,55 @@ up_to_2.4.70() {
   crontab -l | grep -v 'so-playbook-sync_cron' | crontab -
   crontab -l | grep -v 'so-playbook-ruleupdate_cron' | crontab -
 
-  # Check for active Elastalert rules
-  active_rules_count=$(find /opt/so/rules/elastalert/playbook/ -type f -name "*.yaml" | wc -l)
+  if grep -A 1 'playbook:'  /opt/so/saltstack/local/pillar/minions/* | grep -q 'enabled: True'; then
+  
+    # Check for active Elastalert rules
+    active_rules_count=$(find /opt/so/rules/elastalert/playbook/ -type f -name "*.yaml" | wc -l)
 
-  if [[ "$active_rules_count" -gt 0 ]]; then
-      # Prompt the user to AGREE if active Elastalert rules found
-      echo
-      echo "$active_rules_count Active Elastalert/Playbook rules found."
-      echo "In preparation for the new Detections module, they will be backed up and then disabled."
-      echo
-      echo "If you would like to proceed, then type AGREE and press ENTER."
-      echo
-      # Read user input
-      read INPUT
-      if [ "${INPUT^^}" != 'AGREE' ]; then exit 0; fi
+    if [[ "$active_rules_count" -gt 0 ]]; then
+        # Prompt the user to AGREE if active Elastalert rules found
+        echo
+        echo "$active_rules_count Active Elastalert/Playbook rules found."
+        echo "In preparation for the new Detections module, they will be backed up and then disabled."
+        echo
+        echo "If you would like to proceed, then type AGREE and press ENTER."
+        echo
+        # Read user input
+        read INPUT
+        if [ "${INPUT^^}" != 'AGREE' ]; then exit 0; fi
 
-      echo "Backing up the Elastalert rules..."
-      rsync -av --stats /opt/so/rules/elastalert/playbook/*.yaml /nsm/backup/detections-migration/elastalert/
+        echo "Backing up the Elastalert rules..."
+        rsync -av --stats /opt/so/rules/elastalert/playbook/*.yaml /nsm/backup/detections-migration/elastalert/
 
-      # Verify that rsync completed successfully
-      if [[ $? -eq 0 ]]; then
-          # Delete the Elastlaert rules
-          rm -f /opt/so/rules/elastalert/playbook/*.yaml
-          echo "Active Elastalert rules have been backed up."
-      else
-          echo "Error: rsync failed to copy the files. Active Elastalert rules have not been backed up."
-          exit 1
-      fi
+        # Verify that rsync completed successfully
+        if [[ $? -eq 0 ]]; then
+            # Delete the Elastlaert rules
+            rm -f /opt/so/rules/elastalert/playbook/*.yaml
+            echo "Active Elastalert rules have been backed up."
+        else
+            echo "Error: rsync failed to copy the files. Active Elastalert rules have not been backed up."
+            exit 1
+        fi
+    fi
+
+    echo
+    echo "Exporting Sigma rules from Playbook..."
+    MYSQLPW=$(awk '/mysql:/ {print $2}' /opt/so/saltstack/local/pillar/secrets.sls)
+
+    docker exec so-mysql sh -c "exec mysql -uroot -p${MYSQLPW} -D playbook -sN -e \"SELECT id, value FROM custom_values WHERE value LIKE '%View Sigma%'\"" | while read -r id value; do
+        echo -e "$value" > "/nsm/backup/detections-migration/sigma/rules/$id.yaml"
+    done
+
+    echo
+    echo "Exporting Sigma Filters from Playbook..."
+    docker exec so-mysql sh -c "exec mysql -uroot -p${MYSQLPW} -D playbook -sN -e \"SELECT issues.subject as title, custom_values.value as filter FROM issues JOIN custom_values ON issues.id = custom_values.customized_id WHERE custom_values.value LIKE '%sofilter%'\"" > /nsm/backup/detections-migration/sigma/custom-filters.txt
+
+    echo
+    echo "Backing up Playbook database..."
+    docker exec so-mysql sh -c "mysqldump -uroot -p${MYSQLPW} --databases playbook > /tmp/playbook-dump"
+    docker cp so-mysql:/tmp/playbook-dump /nsm/backup/detections-migration/sigma/playbook-dump.sql
   fi
 
-  echo
-  echo "Exporting Sigma rules from Playbook..."
-  MYSQLPW=$(lookup_pillar_secret mysql)
-
-  docker exec so-mysql sh -c "exec mysql -uroot -p${MYSQLPW} -D playbook -sN -e \"SELECT id, value FROM custom_values WHERE value LIKE '%View Sigma%'\"" | while read -r id value; do
-      echo -e "$value" > "/nsm/backup/detections-migration/sigma/rules/$id.yaml"
-  done
-
-  echo
-  echo "Exporting Sigma Filters from Playbook..."
-  docker exec so-mysql sh -c "exec mysql -uroot -p${MYSQLPW} -D playbook -sN -e \"SELECT issues.subject as title, custom_values.value as filter FROM issues JOIN custom_values ON issues.id = custom_values.customized_id WHERE custom_values.value LIKE '%sofilter%'\"" > /nsm/backup/detections-migration/sigma/custom-filters.txt
-
-  echo
-  echo "Backing up Playbook database..."
-  docker exec so-mysql sh -c "mysqldump -uroot -p${MYSQLPW} --databases playbook > /tmp/playbook-dump"
-  docker cp so-mysql:/tmp/playbook-dump /nsm/backup/detections-migration/sigma/playbook-dump.sql
-
   echo
   echo "Stopping Playbook services..."
   so-playbook-stop

From ce0c9f846db5d74afa54727d097ecbb8a24fefb5 Mon Sep 17 00:00:00 2001
From: DefensiveDepth <Josh@defensivedepth.com>
Date: Wed, 27 Mar 2024 16:13:52 -0400
Subject: [PATCH 03/62] Remove containers from so-status

---
 salt/manager/tools/sbin/soup | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup
index 62a579e18..069a4f345 100755
--- a/salt/manager/tools/sbin/soup
+++ b/salt/manager/tools/sbin/soup
@@ -643,6 +643,8 @@ up_to_2.4.70() {
   so-playbook-stop
   so-mysql-stop
   so-soctopus-stop
+  sed -i '/so-playbook\|so-soctopus\|so-mysql/d' /opt/so/conf/so-status/so-status.conf
+
 
   # What about cleaning up various so-utilities like so-playbook-restart?
   echo

From d2c9e0ea4aaa7e7ff7e0aafdf9fa25cbc681eda5 Mon Sep 17 00:00:00 2001
From: DefensiveDepth <Josh@defensivedepth.com>
Date: Thu, 28 Mar 2024 13:04:48 -0400
Subject: [PATCH 04/62] Cleanup

---
 salt/manager/tools/sbin/soup | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup
index 069a4f345..822fa05d2 100755
--- a/salt/manager/tools/sbin/soup
+++ b/salt/manager/tools/sbin/soup
@@ -604,7 +604,7 @@ up_to_2.4.70() {
         echo
         # Read user input
         read INPUT
-        if [ "${INPUT^^}" != 'AGREE' ]; then exit 0; fi
+        if [ "${INPUT^^}" != 'AGREE' ]; then fail "SOUP canceled."; fi
 
         echo "Backing up the Elastalert rules..."
         rsync -av --stats /opt/so/rules/elastalert/playbook/*.yaml /nsm/backup/detections-migration/elastalert/
@@ -615,8 +615,7 @@ up_to_2.4.70() {
             rm -f /opt/so/rules/elastalert/playbook/*.yaml
             echo "Active Elastalert rules have been backed up."
         else
-            echo "Error: rsync failed to copy the files. Active Elastalert rules have not been backed up."
-            exit 1
+            fail "Error: rsync failed to copy the files. Active Elastalert rules have not been backed up."
         fi
     fi
 
@@ -639,14 +638,13 @@ up_to_2.4.70() {
   fi
 
   echo
-  echo "Stopping Playbook services..."
-  so-playbook-stop
-  so-mysql-stop
-  so-soctopus-stop
+  echo "Stopping Playbook services & cleaning up..."
+  docker stop so-playbook 2>/dev/null
+  docker stop so-mysql 2>/dev/null
+  docker stop so-soctopus 2>/dev/null
   sed -i '/so-playbook\|so-soctopus\|so-mysql/d' /opt/so/conf/so-status/so-status.conf
+  rm -f /usr/sbin/so-playbook-* /usr/sbin/so-soctopus-* /usr/sbin/so-mysql-*
 
-
-  # What about cleaning up various so-utilities like so-playbook-restart?
   echo
   echo "Playbook Migration is complete...."
 

From 9c5ba92589e583182373acff0a0eefd00573fea2 Mon Sep 17 00:00:00 2001
From: DefensiveDepth <Josh@defensivedepth.com>
Date: Thu, 28 Mar 2024 13:23:40 -0400
Subject: [PATCH 05/62] Check if container is running first

---
 salt/manager/tools/sbin/soup | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup
index 822fa05d2..4020ec531 100755
--- a/salt/manager/tools/sbin/soup
+++ b/salt/manager/tools/sbin/soup
@@ -639,9 +639,11 @@ up_to_2.4.70() {
 
   echo
   echo "Stopping Playbook services & cleaning up..."
-  docker stop so-playbook 2>/dev/null
-  docker stop so-mysql 2>/dev/null
-  docker stop so-soctopus 2>/dev/null
+  for container in so-playbook so-mysql so-soctopus; do
+      if [ -n "$(docker ps -q -f name=^${container}$)" ]; then
+          docker stop $container
+      fi
+  done
   sed -i '/so-playbook\|so-soctopus\|so-mysql/d' /opt/so/conf/so-status/so-status.conf
   rm -f /usr/sbin/so-playbook-* /usr/sbin/so-soctopus-* /usr/sbin/so-mysql-*
 

From 32b8649c77e0698c987b0b4e01fdeb9dfcb406ad Mon Sep 17 00:00:00 2001
From: DefensiveDepth <Josh@defensivedepth.com>
Date: Thu, 28 Mar 2024 14:31:02 -0400
Subject: [PATCH 06/62] Add more error checking

---
 salt/manager/tools/sbin/soup | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup
index 4020ec531..d5abda783 100755
--- a/salt/manager/tools/sbin/soup
+++ b/salt/manager/tools/sbin/soup
@@ -625,16 +625,16 @@ up_to_2.4.70() {
 
     docker exec so-mysql sh -c "exec mysql -uroot -p${MYSQLPW} -D playbook -sN -e \"SELECT id, value FROM custom_values WHERE value LIKE '%View Sigma%'\"" | while read -r id value; do
         echo -e "$value" > "/nsm/backup/detections-migration/sigma/rules/$id.yaml"
-    done
+    done || fail "Failed to export Sigma rules..."
 
     echo
     echo "Exporting Sigma Filters from Playbook..."
-    docker exec so-mysql sh -c "exec mysql -uroot -p${MYSQLPW} -D playbook -sN -e \"SELECT issues.subject as title, custom_values.value as filter FROM issues JOIN custom_values ON issues.id = custom_values.customized_id WHERE custom_values.value LIKE '%sofilter%'\"" > /nsm/backup/detections-migration/sigma/custom-filters.txt
+    docker exec so-mysql sh -c "exec mysql -uroot -p${MYSQLPW} -D playbook -sN -e \"SELECT issues.subject as title, custom_values.value as filter FROM issues JOIN custom_values ON issues.id = custom_values.customized_id WHERE custom_values.value LIKE '%sofilter%'\"" > /nsm/backup/detections-migration/sigma/custom-filters.txt || fail "Failed to export Custom Sigma Filters."
 
     echo
     echo "Backing up Playbook database..."
-    docker exec so-mysql sh -c "mysqldump -uroot -p${MYSQLPW} --databases playbook > /tmp/playbook-dump"
-    docker cp so-mysql:/tmp/playbook-dump /nsm/backup/detections-migration/sigma/playbook-dump.sql
+    docker exec so-mysql sh -c "mysqldump -uroot -p${MYSQLPW} --databases playbook > /tmp/playbook-dump" || fail "Failed to dump Playbook database."
+    docker cp so-mysql:/tmp/playbook-dump /nsm/backup/detections-migration/sigma/playbook-dump.sql || fail "Failed to backup Playbook database."
   fi
 
   echo

From 3aea2dec85745aca62f89a53cb6f8444629dce32 Mon Sep 17 00:00:00 2001
From: Jason Ertel <jason.ertel@securityonionsolutions.com>
Date: Mon, 1 Apr 2024 09:50:18 -0400
Subject: [PATCH 07/62] analytics

---
 salt/manager/tools/sbin/so-yaml.py      |  82 +++++++++++-
 salt/manager/tools/sbin/so-yaml_test.py | 159 ++++++++++++++++++++++++
 salt/manager/tools/sbin/soup            |  42 +++++++
 salt/soc/config.sls                     |   9 ++
 salt/soc/defaults.yaml                  |   1 +
 salt/soc/enabled.sls                    |   5 +
 salt/soc/files/soc/analytics.js         |   5 +
 salt/soc/files/soc/motd.md              |   4 +
 salt/soc/soc_soc.yaml                   |   5 +
 setup/so-functions                      |   4 +
 setup/so-setup                          |   5 +
 setup/so-whiptail                       |  20 +++
 12 files changed, 339 insertions(+), 2 deletions(-)
 create mode 100644 salt/soc/files/soc/analytics.js

diff --git a/salt/manager/tools/sbin/so-yaml.py b/salt/manager/tools/sbin/so-yaml.py
index 41cab0b23..5427a2e48 100755
--- a/salt/manager/tools/sbin/so-yaml.py
+++ b/salt/manager/tools/sbin/so-yaml.py
@@ -17,13 +17,16 @@ def showUsage(args):
     print('Usage: {} <COMMAND> <YAML_FILE> [ARGS...]'.format(sys.argv[0]))
     print('  General commands:')
     print('    append         - Append a list item to a yaml key, if it exists and is a list. Requires KEY and LISTITEM args.')
+    print('    add            - Add a new key and set its value. Fails if key already exists. Requires KEY and VALUE args.')
     print('    remove         - Removes a yaml key, if it exists. Requires KEY arg.')
+    print('    replace        - Replaces (or adds) a new key and set its value. Requires KEY and VALUE args.')
     print('    help           - Prints this usage information.')
     print('')
     print('  Where:')
     print('   YAML_FILE       - Path to the file that will be modified. Ex: /opt/so/conf/service/conf.yaml')
     print('   KEY             - YAML key, does not support \' or " characters at this time. Ex: level1.level2')
-    print('   LISTITEM        - Item to add to the list.')
+    print('   VALUE           - Value to set for a given key')
+    print('   LISTITEM        - Item to append to a given key\'s list value')
     sys.exit(1)
 
 
@@ -37,6 +40,7 @@ def writeYaml(filename, content):
     file = open(filename, "w")
     return yaml.dump(content, file)
 
+
 def appendItem(content, key, listItem):
     pieces = key.split(".", 1)
     if len(pieces) > 1:
@@ -51,6 +55,30 @@ def appendItem(content, key, listItem):
             print("The key provided does not exist. No action was taken on the file.")
             return 1
 
+
+def convertType(value):
+    if len(value) > 0 and (not value.startswith("0") or len(value) == 1):
+        if "." in value:
+            try:
+                value = float(value)
+                return value
+            except ValueError:
+                pass
+
+        try:
+            value = int(value)
+            return value
+        except ValueError:
+            pass
+
+        lowered_value = value.lower()
+        if lowered_value == "false":
+            return False
+        elif lowered_value == "true":
+            return True
+    return value
+
+
 def append(args):
     if len(args) != 3:
         print('Missing filename, key arg, or list item to append', file=sys.stderr)
@@ -62,11 +90,41 @@ def append(args):
     listItem = args[2]
 
     content = loadYaml(filename)
-    appendItem(content, key, listItem)
+    appendItem(content, key, convertType(listItem))
     writeYaml(filename, content)
 
     return 0
 
+
+def addKey(content, key, value):
+    pieces = key.split(".", 1)
+    if len(pieces) > 1:
+        if not pieces[0] in content:
+            content[pieces[0]] = {}
+        addKey(content[pieces[0]], pieces[1], value)
+    elif key in content:
+        raise KeyError("key already exists")
+    else:
+        content[key] = value
+
+
+def add(args):
+    if len(args) != 3:
+        print('Missing filename, key arg, and/or value', file=sys.stderr)
+        showUsage(None)
+        return
+
+    filename = args[0]
+    key = args[1]
+    value = args[2]
+
+    content = loadYaml(filename)
+    addKey(content, key, convertType(value))
+    writeYaml(filename, content)
+
+    return 0
+
+
 def removeKey(content, key):
     pieces = key.split(".", 1)
     if len(pieces) > 1:
@@ -91,6 +149,24 @@ def remove(args):
     return 0
 
 
+def replace(args):
+    if len(args) != 3:
+        print('Missing filename, key arg, and/or value', file=sys.stderr)
+        showUsage(None)
+        return
+
+    filename = args[0]
+    key = args[1]
+    value = args[2]
+
+    content = loadYaml(filename)
+    removeKey(content, key)
+    addKey(content, key, convertType(value))
+    writeYaml(filename, content)
+
+    return 0
+
+
 def main():
     args = sys.argv[1:]
 
@@ -100,8 +176,10 @@ def main():
 
     commands = {
         "help": showUsage,
+        "add": add,
         "append": append,
         "remove": remove,
+        "replace": replace,
     }
 
     code = 1
diff --git a/salt/manager/tools/sbin/so-yaml_test.py b/salt/manager/tools/sbin/so-yaml_test.py
index 488877ea1..7effabac9 100644
--- a/salt/manager/tools/sbin/so-yaml_test.py
+++ b/salt/manager/tools/sbin/so-yaml_test.py
@@ -42,6 +42,14 @@ class TestRemove(unittest.TestCase):
                 sysmock.assert_called()
                 self.assertIn(mock_stdout.getvalue(), "Usage:")
 
+    def test_remove_missing_arg(self):
+        with patch('sys.exit', new=MagicMock()) as sysmock:
+            with patch('sys.stderr', new=StringIO()) as mock_stdout:
+                sys.argv = ["cmd", "help"]
+                soyaml.remove(["file"])
+                sysmock.assert_called()
+                self.assertIn(mock_stdout.getvalue(), "Missing filename or key arg\n")
+
     def test_remove(self):
         filename = "/tmp/so-yaml_test-remove.yaml"
         file = open(filename, "w")
@@ -106,6 +114,14 @@ class TestRemove(unittest.TestCase):
                 sysmock.assert_called_once_with(1)
                 self.assertIn(mock_stdout.getvalue(), "Missing filename or key arg\n")
 
+    def test_append_missing_arg(self):
+        with patch('sys.exit', new=MagicMock()) as sysmock:
+            with patch('sys.stderr', new=StringIO()) as mock_stdout:
+                sys.argv = ["cmd", "help"]
+                soyaml.append(["file", "key"])
+                sysmock.assert_called()
+                self.assertIn(mock_stdout.getvalue(), "Missing filename, key arg, or list item to append\n")
+
     def test_append(self):
         filename = "/tmp/so-yaml_test-remove.yaml"
         file = open(filename, "w")
@@ -201,3 +217,146 @@ class TestRemove(unittest.TestCase):
                 soyaml.main()
                 sysmock.assert_called()
                 self.assertEqual(mock_stdout.getvalue(), "The existing value for the given key is not a list. No action was taken on the file.\n")
+
+    def test_add_key(self):
+        content = {}
+        soyaml.addKey(content, "foo", 123)
+        self.assertEqual(content, {"foo": 123})
+
+        try:
+            soyaml.addKey(content, "foo", "bar")
+            self.assertFail("expected key error since key already exists")
+        except KeyError:
+            pass
+
+        try:
+            soyaml.addKey(content, "foo.bar", 123)
+            self.assertFail("expected type error since key parent value is not a map")
+        except TypeError:
+            pass
+
+        content = {}
+        soyaml.addKey(content, "foo", "bar")
+        self.assertEqual(content, {"foo": "bar"})
+
+        soyaml.addKey(content, "badda.badda", "boom")
+        self.assertEqual(content, {"foo": "bar", "badda": {"badda": "boom"}})
+
+    def test_add_missing_arg(self):
+        with patch('sys.exit', new=MagicMock()) as sysmock:
+            with patch('sys.stderr', new=StringIO()) as mock_stdout:
+                sys.argv = ["cmd", "help"]
+                soyaml.add(["file", "key"])
+                sysmock.assert_called()
+                self.assertIn(mock_stdout.getvalue(), "Missing filename, key arg, and/or value\n")
+
+    def test_add(self):
+        filename = "/tmp/so-yaml_test-add.yaml"
+        file = open(filename, "w")
+        file.write("{key1: { child1: 123, child2: abc }, key2: false, key3: [a,b,c]}")
+        file.close()
+
+        soyaml.add([filename, "key4", "d"])
+
+        file = open(filename, "r")
+        actual = file.read()
+        file.close()
+        expected = "key1:\n  child1: 123\n  child2: abc\nkey2: false\nkey3:\n- a\n- b\n- c\nkey4: d\n"
+        self.assertEqual(actual, expected)
+
+    def test_add_nested(self):
+        filename = "/tmp/so-yaml_test-add.yaml"
+        file = open(filename, "w")
+        file.write("{key1: { child1: 123, child2: [a,b,c] }, key2: false, key3: [e,f,g]}")
+        file.close()
+
+        soyaml.add([filename, "key1.child3", "d"])
+
+        file = open(filename, "r")
+        actual = file.read()
+        file.close()
+
+        expected = "key1:\n  child1: 123\n  child2:\n  - a\n  - b\n  - c\n  child3: d\nkey2: false\nkey3:\n- e\n- f\n- g\n"
+        self.assertEqual(actual, expected)
+
+    def test_add_nested_deep(self):
+        filename = "/tmp/so-yaml_test-add.yaml"
+        file = open(filename, "w")
+        file.write("{key1: { child1: 123, child2: { deep1: 45 } }, key2: false, key3: [e,f,g]}")
+        file.close()
+
+        soyaml.add([filename, "key1.child2.deep2", "d"])
+
+        file = open(filename, "r")
+        actual = file.read()
+        file.close()
+
+        expected = "key1:\n  child1: 123\n  child2:\n    deep1: 45\n    deep2: d\nkey2: false\nkey3:\n- e\n- f\n- g\n"
+        self.assertEqual(actual, expected)
+
+    def test_replace_missing_arg(self):
+        with patch('sys.exit', new=MagicMock()) as sysmock:
+            with patch('sys.stderr', new=StringIO()) as mock_stdout:
+                sys.argv = ["cmd", "help"]
+                soyaml.replace(["file", "key"])
+                sysmock.assert_called()
+                self.assertIn(mock_stdout.getvalue(), "Missing filename, key arg, and/or value\n")
+
+    def test_replace(self):
+        filename = "/tmp/so-yaml_test-add.yaml"
+        file = open(filename, "w")
+        file.write("{key1: { child1: 123, child2: abc }, key2: false, key3: [a,b,c]}")
+        file.close()
+
+        soyaml.replace([filename, "key2", True])
+
+        file = open(filename, "r")
+        actual = file.read()
+        file.close()
+        expected = "key1:\n  child1: 123\n  child2: abc\nkey2: true\nkey3:\n- a\n- b\n- c\n"
+        self.assertEqual(actual, expected)
+
+    def test_replace_nested(self):
+        filename = "/tmp/so-yaml_test-add.yaml"
+        file = open(filename, "w")
+        file.write("{key1: { child1: 123, child2: [a,b,c] }, key2: false, key3: [e,f,g]}")
+        file.close()
+
+        soyaml.replace([filename, "key1.child2", "d"])
+
+        file = open(filename, "r")
+        actual = file.read()
+        file.close()
+
+        expected = "key1:\n  child1: 123\n  child2: d\nkey2: false\nkey3:\n- e\n- f\n- g\n"
+        self.assertEqual(actual, expected)
+
+    def test_replace_nested_deep(self):
+        filename = "/tmp/so-yaml_test-add.yaml"
+        file = open(filename, "w")
+        file.write("{key1: { child1: 123, child2: { deep1: 45 } }, key2: false, key3: [e,f,g]}")
+        file.close()
+
+        soyaml.replace([filename, "key1.child2.deep1", 46])
+
+        file = open(filename, "r")
+        actual = file.read()
+        file.close()
+
+        expected = "key1:\n  child1: 123\n  child2:\n    deep1: 46\nkey2: false\nkey3:\n- e\n- f\n- g\n"
+        self.assertEqual(actual, expected)
+
+    def test_convert(self):
+        self.assertEqual(soyaml.convertType("foo"), "foo")
+        self.assertEqual(soyaml.convertType("foo.bar"), "foo.bar")
+        self.assertEqual(soyaml.convertType("123"), 123)
+        self.assertEqual(soyaml.convertType("0"), 0)
+        self.assertEqual(soyaml.convertType("00"), "00")
+        self.assertEqual(soyaml.convertType("0123"), "0123")
+        self.assertEqual(soyaml.convertType("123.456"), 123.456)
+        self.assertEqual(soyaml.convertType("0123.456"), "0123.456")
+        self.assertEqual(soyaml.convertType("true"), True)
+        self.assertEqual(soyaml.convertType("TRUE"), True)
+        self.assertEqual(soyaml.convertType("false"), False)
+        self.assertEqual(soyaml.convertType("FALSE"), False)
+        self.assertEqual(soyaml.convertType(""), "")
diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup
index a585f877c..db5335a7a 100755
--- a/salt/manager/tools/sbin/soup
+++ b/salt/manager/tools/sbin/soup
@@ -357,6 +357,7 @@ preupgrade_changes() {
     [[ "$INSTALLEDVERSION" == 2.4.30 ]] && up_to_2.4.40
     [[ "$INSTALLEDVERSION" == 2.4.40 ]] && up_to_2.4.50
     [[ "$INSTALLEDVERSION" == 2.4.50 ]] && up_to_2.4.60
+    [[ "$INSTALLEDVERSION" == 2.4.60 ]] && up_to_2.4.70
     true
 }
 
@@ -373,6 +374,7 @@ postupgrade_changes() {
     [[ "$POSTVERSION" == 2.4.30 ]] && post_to_2.4.40
     [[ "$POSTVERSION"  == 2.4.40 ]] && post_to_2.4.50
     [[ "$POSTVERSION"  == 2.4.50 ]] && post_to_2.4.60
+    [[ "$POSTVERSION"  == 2.4.60 ]] && post_to_2.4.70
     true
 }
 
@@ -435,6 +437,11 @@ post_to_2.4.60() {
   POSTVERSION=2.4.60
 }
 
+post_to_2.4.70() {
+  echo "Nothing to apply"
+  POSTVERSION=2.4.70
+}
+
 repo_sync() {
   echo "Sync the local repo."
   su socore -c '/usr/sbin/so-repo-sync' || fail "Unable to complete so-repo-sync."
@@ -574,6 +581,41 @@ up_to_2.4.60() {
   INSTALLEDVERSION=2.4.60
 }
 
+up_to_2.4.70() {
+  if [[ -z $UNATTENDED && $is_airgap -ne 0 ]]; then
+    cat << ASSIST_EOF
+
+--------------- SOC Telemetry ---------------
+
+The Security Onion development team could use your help! Enabling SOC
+Telemetry will help the team understand which UI features are being
+used and enables informed prioritization of future development.
+
+Adjust this setting at anytime via the SOC Configuration screen.
+
+For more information visit https://docs.securityonion.net/telemetry.rst.
+
+ASSIST_EOF
+
+    echo -n "Continue the upgrade with SOC Telemetry enabled [Y/n]? "
+  
+    read -r input
+    input=$(echo "${input,,}" | xargs echo -n)
+    echo ""
+    if [[ ${#input} -eq 0 || "$input" == "yes" || "$input" == "y" || "$input" == "yy" ]]; then
+      echo "Thank you for helping improve Security Onion!"
+    else
+      if so-yaml.py replace /opt/so/saltstack/local/pillar/soc/soc_soc.sls soc.telemetryEnabled false; then
+        echo "Disabled SOC Telemetry."
+      else
+        fail "Failed to disable SOC Telemetry; aborting."
+      fi
+    fi
+    echo ""
+  fi
+  INSTALLEDVERSION=2.4.70
+}
+
 determine_elastic_agent_upgrade() {
   if [[ $is_airgap -eq 0 ]]; then
     update_elastic_agent_airgap
diff --git a/salt/soc/config.sls b/salt/soc/config.sls
index ad0ab1c8d..3e756f977 100644
--- a/salt/soc/config.sls
+++ b/salt/soc/config.sls
@@ -52,6 +52,15 @@ socsaltdir:
     - mode: 770
     - makedirs: True
 
+socanalytics:
+  file.managed:
+    - name: /opt/so/conf/soc/analytics.js
+    - source: salt://soc/files/soc/analytics.js
+    - user: 939
+    - group: 939
+    - mode: 600
+    - show_changes: False
+
 socconfig:
   file.managed:
     - name: /opt/so/conf/soc/soc.json
diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml
index 861f6b02c..2ba99cd11 100644
--- a/salt/soc/defaults.yaml
+++ b/salt/soc/defaults.yaml
@@ -1,5 +1,6 @@
 soc:
   enabled: False
+  telemetryEnabled: true
   config:
     logFilename: /opt/sensoroni/logs/sensoroni-server.log
     logLevel: info
diff --git a/salt/soc/enabled.sls b/salt/soc/enabled.sls
index bbe36e5b7..6cea0c70d 100644
--- a/salt/soc/enabled.sls
+++ b/salt/soc/enabled.sls
@@ -8,6 +8,7 @@
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 {%   from 'docker/docker.map.jinja' import DOCKER %}
 {%   from 'soc/merged.map.jinja' import DOCKER_EXTRA_HOSTS %}
+{%   from 'soc/merged.map.jinja' import SOCMERGED %}
 
 include:
   - soc.config
@@ -31,6 +32,9 @@ so-soc:
       - /nsm/soc/uploads:/nsm/soc/uploads:rw
       - /opt/so/log/soc/:/opt/sensoroni/logs/:rw
       - /opt/so/conf/soc/soc.json:/opt/sensoroni/sensoroni.json:ro
+{% if SOCMERGED.telemetryEnabled and not GLOBALS.airgap %}
+      - /opt/so/conf/soc/analytics.js:/opt/sensoroni/html/js/analytics.js:ro
+{% endif %}
       - /opt/so/conf/soc/motd.md:/opt/sensoroni/html/motd.md:ro
       - /opt/so/conf/soc/banner.md:/opt/sensoroni/html/login/banner.md:ro
       - /opt/so/conf/soc/sigma_so_pipeline.yaml:/opt/sensoroni/sigma_so_pipeline.yaml:ro
@@ -67,6 +71,7 @@ so-soc:
       - file: socdatadir
       - file: soclogdir
       - file: socconfig
+      - file: socanalytics
       - file: socmotd
       - file: socbanner
       - file: soccustom
diff --git a/salt/soc/files/soc/analytics.js b/salt/soc/files/soc/analytics.js
new file mode 100644
index 000000000..6a0d72d5d
--- /dev/null
+++ b/salt/soc/files/soc/analytics.js
@@ -0,0 +1,5 @@
+(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':
+  new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],
+  j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=
+  'https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);
+  })(window,document,'script','dataLayer','GTM-TM46SL7T');
diff --git a/salt/soc/files/soc/motd.md b/salt/soc/files/soc/motd.md
index d6b0d3d27..005a2be0f 100644
--- a/salt/soc/files/soc/motd.md
+++ b/salt/soc/files/soc/motd.md
@@ -12,6 +12,10 @@ To see all the latest features and fixes in this version of Security Onion, clic
 
 Want the best hardware for your enterprise deployment? Check out our [enterprise appliances](https://securityonionsolutions.com/hardware/)!
 
+## Premium Support
+
+Experiencing difficulties and need priority support or remote assistance? We offer a [premium support plan](https://securityonionsolutions.com/support/) to assist corporate, educational, and government organizations.
+
 ## Customize This Space
 
 Make this area your own by customizing the content in the [Config](/#/config?s=soc.files.soc.motd__md) interface.
diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml
index eae52e31b..eed0113fc 100644
--- a/salt/soc/soc_soc.yaml
+++ b/salt/soc/soc_soc.yaml
@@ -2,6 +2,11 @@ soc:
   enabled:
     description: You can enable or disable SOC.
     advanced: True
+  telemetryEnabled:
+    title: SOC Telemetry
+    description: When enabled, SOC provides feature usage data to the Security Onion development team via Google Analytics. This data helps Security Onion developers determine which product features are being used and can also provide insight into improving the user interface. When changing this setting, wait for the grid to fully synchronize and then perform a hard browser refresh on SOC, to force the browser cache to update and reflect the new setting.
+    global: True
+    helpLink: telemetry.html
   files:
     soc:
       banner__md:
diff --git a/setup/so-functions b/setup/so-functions
index 0d66a2621..3a0da7bda 100755
--- a/setup/so-functions
+++ b/setup/so-functions
@@ -1258,6 +1258,10 @@ soc_pillar() {
 		"    server:"\
 		"      srvKey: '$SOCSRVKEY'"\
 		"" > "$soc_pillar_file"
+
+	if [[ $telemetry -ne 0 ]]; then
+		echo "  telemetryEnabled: false" >> $soc_pillar_file
+	fi
 }
 
 telegraf_pillar() {
diff --git a/setup/so-setup b/setup/so-setup
index 2f62dca78..fc13e5b18 100755
--- a/setup/so-setup
+++ b/setup/so-setup
@@ -447,6 +447,7 @@ if ! [[ -f $install_opt_file ]]; then
 		get_redirect
 		# Does the user want to allow access to the UI?
 		collect_so_allow
+		whiptail_accept_telemetry
 		whiptail_end_settings
 	elif [[ $is_standalone ]]; then
 		waitforstate=true
@@ -468,6 +469,7 @@ if ! [[ -f $install_opt_file ]]; then
 		collect_webuser_inputs
 		get_redirect
 		collect_so_allow
+		whiptail_accept_telemetry
 		whiptail_end_settings
 	elif [[ $is_manager ]]; then
 		info "Setting up as node type manager"
@@ -488,6 +490,7 @@ if ! [[ -f $install_opt_file ]]; then
 		collect_webuser_inputs
 		get_redirect
 		collect_so_allow
+		whiptail_accept_telemetry
 		whiptail_end_settings
 	elif [[ $is_managersearch ]]; then
 		info "Setting up as node type managersearch"
@@ -508,6 +511,7 @@ if ! [[ -f $install_opt_file ]]; then
 		collect_webuser_inputs
 		get_redirect
 		collect_so_allow
+		whiptail_accept_telemetry
 		whiptail_end_settings
 	elif [[ $is_sensor ]]; then
 		info "Setting up as node type sensor"
@@ -597,6 +601,7 @@ if ! [[ -f $install_opt_file ]]; then
 		collect_webuser_inputs
 		get_redirect
 		collect_so_allow
+		whiptail_accept_telemetry
 		whiptail_end_settings
 
 	elif [[ $is_receiver ]]; then
diff --git a/setup/so-whiptail b/setup/so-whiptail
index 904654c9b..95b21ccde 100755
--- a/setup/so-whiptail
+++ b/setup/so-whiptail
@@ -144,6 +144,26 @@ whiptail_cancel() {
 	exit 1
 }
 
+whiptail_accept_telemetry() {
+
+	[ -n "$TESTING" ] && return
+
+	read -r -d '' message <<- EOM
+
+	The Security Onion development team could use your help! Enabling SOC
+	Telemetry will help the team understand which UI features are being
+	used and enables informed prioritization of future development.
+
+	Adjust this setting at anytime via the SOC Configuration screen.
+
+	For more information visit https://docs.securityonion.net/telemetry.rst.
+
+	Enable SOC Telemetry to help improve future releases?
+	EOM
+	whiptail --title "$whiptail_title" --yesno "$message" 13 75
+	telemetry=$?
+}
+
 whiptail_check_exitstatus() {
 	case $1 in
 		1)

From c1bf710e463c15428be7d1ec3c44adaec2e7fa15 Mon Sep 17 00:00:00 2001
From: Jason Ertel <jason.ertel@securityonionsolutions.com>
Date: Mon, 1 Apr 2024 10:32:25 -0400
Subject: [PATCH 08/62] limit col size

---
 setup/so-whiptail | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup/so-whiptail b/setup/so-whiptail
index 95b21ccde..b4928eb98 100755
--- a/setup/so-whiptail
+++ b/setup/so-whiptail
@@ -156,7 +156,7 @@ whiptail_accept_telemetry() {
 
 	Adjust this setting at anytime via the SOC Configuration screen.
 
-	For more information visit https://docs.securityonion.net/telemetry.rst.
+	Additional information: https://docs.securityonion.net/telemetry.rst.
 
 	Enable SOC Telemetry to help improve future releases?
 	EOM

From 2c68fd6311eb90672ea5d3c515ee496594a451d8 Mon Sep 17 00:00:00 2001
From: Jason Ertel <jason.ertel@securityonionsolutions.com>
Date: Mon, 1 Apr 2024 10:32:54 -0400
Subject: [PATCH 09/62] limit col size

---
 setup/so-whiptail | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup/so-whiptail b/setup/so-whiptail
index b4928eb98..77e12c63f 100755
--- a/setup/so-whiptail
+++ b/setup/so-whiptail
@@ -160,7 +160,7 @@ whiptail_accept_telemetry() {
 
 	Enable SOC Telemetry to help improve future releases?
 	EOM
-	whiptail --title "$whiptail_title" --yesno "$message" 13 75
+	whiptail --title "$whiptail_title" --yesno "$message" 15 75
 	telemetry=$?
 }
 

From ff777560ac88abb527cd6b3ef14a3d12bed9eda6 Mon Sep 17 00:00:00 2001
From: Jason Ertel <jason.ertel@securityonionsolutions.com>
Date: Mon, 1 Apr 2024 10:35:15 -0400
Subject: [PATCH 10/62] limit col size

---
 setup/so-whiptail | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/setup/so-whiptail b/setup/so-whiptail
index 77e12c63f..5fa5bf343 100755
--- a/setup/so-whiptail
+++ b/setup/so-whiptail
@@ -451,6 +451,12 @@ whiptail_end_settings() {
 		done
 	fi
 
+	if [[ $telemetry -eq 0 ]]; then
+		__append_end_msg "SOC Telemetry: enabled"
+	else
+		__append_end_msg "SOC Telemetry: disabled"
+	fi
+
 	# ADVANCED
 	if [[ $MANAGERADV == 'ADVANCED' ]]; then
 		__append_end_msg "Advanced Manager Settings:"

From f17d8d3369f136615d078f457efe6b52545b3398 Mon Sep 17 00:00:00 2001
From: Jason Ertel <jason.ertel@securityonionsolutions.com>
Date: Mon, 1 Apr 2024 10:59:44 -0400
Subject: [PATCH 11/62] analytics

---
 salt/manager/tools/sbin/soup | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup
index db5335a7a..30a170ea7 100755
--- a/salt/manager/tools/sbin/soup
+++ b/salt/manager/tools/sbin/soup
@@ -593,7 +593,7 @@ used and enables informed prioritization of future development.
 
 Adjust this setting at anytime via the SOC Configuration screen.
 
-For more information visit https://docs.securityonion.net/telemetry.rst.
+Additional information: https://docs.securityonion.net/telemetry.rst.
 
 ASSIST_EOF
 

From f0835586662bb7fd1072a205f1ac1803e6699a8d Mon Sep 17 00:00:00 2001
From: Jason Ertel <jason.ertel@securityonionsolutions.com>
Date: Tue, 2 Apr 2024 09:42:43 -0400
Subject: [PATCH 12/62] break out into sep func

---
 salt/manager/tools/sbin/soup | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup
index 30a170ea7..a71775501 100755
--- a/salt/manager/tools/sbin/soup
+++ b/salt/manager/tools/sbin/soup
@@ -582,6 +582,11 @@ up_to_2.4.60() {
 }
 
 up_to_2.4.70() {
+  toggle_telemetry()
+  INSTALLEDVERSION=2.4.70
+}
+
+toggle_telemetry() {
   if [[ -z $UNATTENDED && $is_airgap -ne 0 ]]; then
     cat << ASSIST_EOF
 
@@ -613,7 +618,6 @@ ASSIST_EOF
     fi
     echo ""
   fi
-  INSTALLEDVERSION=2.4.70
 }
 
 determine_elastic_agent_upgrade() {

From 3b112e20e3129d65a98161d22c998b9cd46dd3af Mon Sep 17 00:00:00 2001
From: Jason Ertel <jason.ertel@securityonionsolutions.com>
Date: Tue, 2 Apr 2024 12:32:33 -0400
Subject: [PATCH 13/62] fix syntax error

---
 salt/manager/tools/sbin/soup | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup
index a71775501..455fd2256 100755
--- a/salt/manager/tools/sbin/soup
+++ b/salt/manager/tools/sbin/soup
@@ -582,7 +582,7 @@ up_to_2.4.60() {
 }
 
 up_to_2.4.70() {
-  toggle_telemetry()
+  toggle_telemetry
   INSTALLEDVERSION=2.4.70
 }
 

From 283939b18a23884f5cd3aa175a701cc99556cef4 Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Tue, 2 Apr 2024 15:36:01 -0400
Subject: [PATCH 14/62] Gather metrics from elastic agent to influx

---
 salt/manager/init.sls                         | 20 +++++++++++++
 .../manager/tools/sbin/so-elasticagent-status | 10 +++++++
 salt/telegraf/scripts/agentstatus.sh          | 30 +++++++++++++++++++
 3 files changed, 60 insertions(+)
 create mode 100644 salt/manager/tools/sbin/so-elasticagent-status
 create mode 100644 salt/telegraf/scripts/agentstatus.sh

diff --git a/salt/manager/init.sls b/salt/manager/init.sls
index 0ff4fa85a..d979482ef 100644
--- a/salt/manager/init.sls
+++ b/salt/manager/init.sls
@@ -27,6 +27,15 @@ repo_log_dir:
       - user
       - group
 
+agents_log_dir:
+  file.directory:
+    - name: /opt/so/log/agents
+    - user: root
+    - group: root
+    - recurse:
+      - user
+      - group
+
 yara_log_dir:
   file.directory:
     - name: /opt/so/log/yarasync
@@ -101,6 +110,17 @@ so-repo-sync:
     - hour: '{{ MANAGERMERGED.reposync.hour }}'
     - minute: '{{ MANAGERMERGED.reposync.minute }}'
 
+so_fleetagent_status:
+  cron.present:
+    - name: /usr/sbin/so-elasticagent-status > /opt/so/log/agents/agentstatus.log 2>&1
+    - identifier: so_fleetagent_status
+    - user: root
+    - minute: '*/5'
+    - hour: '*'
+    - daymonth: '*'
+    - month: '*'
+    - dayweek: '*'
+
 socore_own_saltstack:
   file.directory:
     - name: /opt/so/saltstack
diff --git a/salt/manager/tools/sbin/so-elasticagent-status b/salt/manager/tools/sbin/so-elasticagent-status
new file mode 100644
index 000000000..dffd76660
--- /dev/null
+++ b/salt/manager/tools/sbin/so-elasticagent-status
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+. /usr/sbin/so-common
+
+curl -s -K /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/agent_status" | jq .
\ No newline at end of file
diff --git a/salt/telegraf/scripts/agentstatus.sh b/salt/telegraf/scripts/agentstatus.sh
new file mode 100644
index 000000000..1673e7bd2
--- /dev/null
+++ b/salt/telegraf/scripts/agentstatus.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+#
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+
+
+# if this script isn't already running
+if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then
+
+    LOGFILE=$(cat /opt/so/log/agents/agentstatus.log)
+    ONLINE=$(cat $LOGFILE | grep -wF online | awk '{print $2}' | tr -d ',')
+    ERROR=$(cat $LOGFILE | grep -wF error | awk '{print $2}' | tr -d ',')
+    INACTIVE=$(cat $LOGFILE | grep -wF inactive | awk '{print $2}' | tr -d ',')
+    OFFLINE=$(cat $LOGFILE | grep -wF offline | awk '{print $2}' | tr -d ',')
+    UPDATING=$(cat $LOGFILE | grep -wF updating | awk '{print $2}' | tr -d ',')
+    UNENROLLED=$(cat $LOGFILE | grep -wF unenrolled | awk '{print $2}' | tr -d ',')
+    OTHER=$(cat $LOGFILE | grep -wF other | awk '{print $2}' | tr -d ',')
+    EVENTS=$(cat $LOGFILE | grep -wF events | awk '{print $2}' | tr -d ',')
+    TOTAL=$(cat $LOGFILE | grep -wF total | awk '{print $2}' | tr -d ',')
+    ALL=$(cat $LOGFILE | grep -wF all | awk '{print $2}' | tr -d ',')
+    ACTIVE=$(cat $LOGFILE | grep -wF active | awk '{print $2}')
+
+    echo "agentstatus online=$ONLINE,error=$ERROR,inactive=$INACTIVE,offline=$OFFLINE,updating=$UPDATING,unenrolled=$UNENROLLED,other=$OTHER,events=$EVENTS,total=$TOTAL,all=$ALL,active=$ACTIVE"
+    
+fi
+
+exit 0

From b6187ab76992f1e37328b58c0d017952706df359 Mon Sep 17 00:00:00 2001
From: Jason Ertel <jason.ertel@securityonionsolutions.com>
Date: Tue, 2 Apr 2024 15:54:39 -0400
Subject: [PATCH 15/62] Improve wording for Airgap annotation

---
 salt/global/soc_global.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/global/soc_global.yaml b/salt/global/soc_global.yaml
index a48476214..910babcf4 100644
--- a/salt/global/soc_global.yaml
+++ b/salt/global/soc_global.yaml
@@ -28,7 +28,7 @@ global:
     description: Used for handling of authentication cookies. 
     global: True
   airgap:
-    description: Sets airgap mode.
+    description: Airgapped systems do not have network connectivity to the internet. This setting represents how this grid was configured during initial setup. While it is technically possible to manually switch systems between airgap and non-airgap, there are some nuances and additional steps involved. For that reason this setting is marked read-only. Contact your support representative for guidance if there is a need to chane this setting.
     global: True
     readonly: True
   imagerepo:

From 1ee2a6d37b41c248b0261cd8f7681a7a517a6dab Mon Sep 17 00:00:00 2001
From: Jason Ertel <jason.ertel@securityonionsolutions.com>
Date: Wed, 3 Apr 2024 08:21:30 -0400
Subject: [PATCH 16/62] Improve wording for Airgap annotation

---
 salt/global/soc_global.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/global/soc_global.yaml b/salt/global/soc_global.yaml
index 910babcf4..daaf28b28 100644
--- a/salt/global/soc_global.yaml
+++ b/salt/global/soc_global.yaml
@@ -28,7 +28,7 @@ global:
     description: Used for handling of authentication cookies. 
     global: True
   airgap:
-    description: Airgapped systems do not have network connectivity to the internet. This setting represents how this grid was configured during initial setup. While it is technically possible to manually switch systems between airgap and non-airgap, there are some nuances and additional steps involved. For that reason this setting is marked read-only. Contact your support representative for guidance if there is a need to chane this setting.
+    description: Airgapped systems do not have network connectivity to the internet. This setting represents how this grid was configured during initial setup. While it is technically possible to manually switch systems between airgap and non-airgap, there are some nuances and additional steps involved. For that reason this setting is marked read-only. Contact your support representative for guidance if there is a need to change this setting.
     global: True
     readonly: True
   imagerepo:

From a8f25150f62a34e2a05138139ce1bdc68a52d85e Mon Sep 17 00:00:00 2001
From: DefensiveDepth <Josh@defensivedepth.com>
Date: Wed, 3 Apr 2024 08:21:50 -0400
Subject: [PATCH 17/62] Feature - auto-enabled Sigma rules

---
 salt/soc/defaults.yaml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml
index 8b6bceef0..1d0eb0e38 100644
--- a/salt/soc/defaults.yaml
+++ b/salt/soc/defaults.yaml
@@ -1216,6 +1216,10 @@ soc:
         elastalertengine:
           allowRegex: ''
           autoUpdateEnabled: true
+          autoEnabledSigmaRules:
+            - core+critical
+            - securityonion-resources+critical
+            - securityonion-resources+high
           communityRulesImportFrequencySeconds: 86400
           denyRegex: ''
           elastAlertRulesFolder: /opt/sensoroni/elastalert

From c1b5ef08917307e677a1c2b555478c8836895790 Mon Sep 17 00:00:00 2001
From: Jason Ertel <jason.ertel@securityonionsolutions.com>
Date: Wed, 3 Apr 2024 08:44:40 -0400
Subject: [PATCH 18/62] ensure so-yaml.py is updated during soup

---
 salt/common/soup_scripts.sls | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/salt/common/soup_scripts.sls b/salt/common/soup_scripts.sls
index fd32b8a28..6e93954f0 100644
--- a/salt/common/soup_scripts.sls
+++ b/salt/common/soup_scripts.sls
@@ -68,3 +68,10 @@ copy_so-firewall_sbin:
     - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-firewall
     - force: True
     - preserve: True
+
+copy_so-yaml_sbin:
+  file.copy:
+    - name: /usr/sbin/so-yaml.py
+    - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-yaml.py
+    - force: True
+    - preserve: True

From 0efdcfcb52becf95fe6e505c93f07175ea2785fb Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Wed, 3 Apr 2024 09:36:02 -0400
Subject: [PATCH 19/62] add agentstatus to telegraf

---
 salt/telegraf/etc/telegraf.conf | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf
index 1c5801645..9ced1317c 100644
--- a/salt/telegraf/etc/telegraf.conf
+++ b/salt/telegraf/etc/telegraf.conf
@@ -202,6 +202,16 @@
   insecure_skip_verify = true
 {%- endif %}
 
+# # Get Agent Stats
+{%- if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone', 'so-import']   %}
+[[inputs.exec]]
+   commands = [
+      "/scripts/agentstatus.sh"
+   ]
+  data_format = "influx"
+  timeout = "15s"
+{%- endif %}
+
 #   ## Timeout for HTTP requests to the elastic search server(s)
 #   http_timeout = "5s"
 #

From 0dfde3c9f230f4b76208822cc05d910c0797b9f1 Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Wed, 3 Apr 2024 09:40:14 -0400
Subject: [PATCH 20/62] add agentstatus to telegraf

---
 salt/telegraf/defaults.yaml     |  4 ++++
 salt/telegraf/etc/telegraf.conf | 10 ----------
 2 files changed, 4 insertions(+), 10 deletions(-)

diff --git a/salt/telegraf/defaults.yaml b/salt/telegraf/defaults.yaml
index d3718f0b5..3cf2f731d 100644
--- a/salt/telegraf/defaults.yaml
+++ b/salt/telegraf/defaults.yaml
@@ -11,6 +11,7 @@ telegraf:
     quiet: 'false'
   scripts:
     eval:
+      - agentstatus.sh
       - checkfiles.sh
       - influxdbsize.sh
       - lasthighstate.sh
@@ -23,6 +24,7 @@ telegraf:
       - zeekcaptureloss.sh
       - zeekloss.sh
     standalone:
+      - agentstatus.sh
       - checkfiles.sh
       - eps.sh
       - influxdbsize.sh
@@ -38,6 +40,7 @@ telegraf:
       - zeekloss.sh
       - features.sh
     manager:
+      - agentstatus.sh
       - influxdbsize.sh
       - lasthighstate.sh
       - os.sh
@@ -46,6 +49,7 @@ telegraf:
       - sostatus.sh
       - features.sh
     managersearch:
+      - agentstatus.sh
       - eps.sh
       - influxdbsize.sh
       - lasthighstate.sh
diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf
index 9ced1317c..1c5801645 100644
--- a/salt/telegraf/etc/telegraf.conf
+++ b/salt/telegraf/etc/telegraf.conf
@@ -202,16 +202,6 @@
   insecure_skip_verify = true
 {%- endif %}
 
-# # Get Agent Stats
-{%- if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone', 'so-import']   %}
-[[inputs.exec]]
-   commands = [
-      "/scripts/agentstatus.sh"
-   ]
-  data_format = "influx"
-  timeout = "15s"
-{%- endif %}
-
 #   ## Timeout for HTTP requests to the elastic search server(s)
 #   http_timeout = "5s"
 #

From 513273c8c34868355fc9ca12d34c3dbfe74c8c8c Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Wed, 3 Apr 2024 09:43:55 -0400
Subject: [PATCH 21/62] add agentstatus to telegraf

---
 salt/telegraf/enabled.sls            | 1 +
 salt/telegraf/scripts/agentstatus.sh | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/salt/telegraf/enabled.sls b/salt/telegraf/enabled.sls
index b1fa0c247..8e3bc9fbe 100644
--- a/salt/telegraf/enabled.sls
+++ b/salt/telegraf/enabled.sls
@@ -56,6 +56,7 @@ so-telegraf:
       - /opt/so/log/raid:/var/log/raid:ro
       - /opt/so/log/sostatus:/var/log/sostatus:ro
       - /opt/so/log/salt:/var/log/salt:ro
+      - /opt/so/log/agents:/var/log/agents:ro
       {% if DOCKER.containers['so-telegraf'].custom_bind_mounts %}
         {% for BIND in DOCKER.containers['so-telegraf'].custom_bind_mounts %}
       - {{ BIND }}
diff --git a/salt/telegraf/scripts/agentstatus.sh b/salt/telegraf/scripts/agentstatus.sh
index 1673e7bd2..ccba213f6 100644
--- a/salt/telegraf/scripts/agentstatus.sh
+++ b/salt/telegraf/scripts/agentstatus.sh
@@ -10,7 +10,7 @@
 # if this script isn't already running
 if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then
 
-    LOGFILE=$(cat /opt/so/log/agents/agentstatus.log)
+    LOGFILE=$(cat /var/log/agents/agentstatus.log)
     ONLINE=$(cat $LOGFILE | grep -wF online | awk '{print $2}' | tr -d ',')
     ERROR=$(cat $LOGFILE | grep -wF error | awk '{print $2}' | tr -d ',')
     INACTIVE=$(cat $LOGFILE | grep -wF inactive | awk '{print $2}' | tr -d ',')

From 333561236588306871f1d4dd605c816cdabd2a53 Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Wed, 3 Apr 2024 09:54:16 -0400
Subject: [PATCH 22/62] add agentstatus to telegraf

---
 salt/telegraf/scripts/agentstatus.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/telegraf/scripts/agentstatus.sh b/salt/telegraf/scripts/agentstatus.sh
index ccba213f6..7ba599e8a 100644
--- a/salt/telegraf/scripts/agentstatus.sh
+++ b/salt/telegraf/scripts/agentstatus.sh
@@ -10,7 +10,7 @@
 # if this script isn't already running
 if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then
 
-    LOGFILE=$(cat /var/log/agents/agentstatus.log)
+    LOGFILE=$(/var/log/agents/agentstatus.log)
     ONLINE=$(cat $LOGFILE | grep -wF online | awk '{print $2}' | tr -d ',')
     ERROR=$(cat $LOGFILE | grep -wF error | awk '{print $2}' | tr -d ',')
     INACTIVE=$(cat $LOGFILE | grep -wF inactive | awk '{print $2}' | tr -d ',')

From 64748b98adf4bdf32a17a2069487d7686c842ea1 Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Wed, 3 Apr 2024 09:56:12 -0400
Subject: [PATCH 23/62] add agentstatus to telegraf

---
 salt/telegraf/scripts/agentstatus.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/telegraf/scripts/agentstatus.sh b/salt/telegraf/scripts/agentstatus.sh
index 7ba599e8a..f57fc5f20 100644
--- a/salt/telegraf/scripts/agentstatus.sh
+++ b/salt/telegraf/scripts/agentstatus.sh
@@ -10,7 +10,7 @@
 # if this script isn't already running
 if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then
 
-    LOGFILE=$(/var/log/agents/agentstatus.log)
+    LOGFILE=/var/log/agents/agentstatus.log
     ONLINE=$(cat $LOGFILE | grep -wF online | awk '{print $2}' | tr -d ',')
     ERROR=$(cat $LOGFILE | grep -wF error | awk '{print $2}' | tr -d ',')
     INACTIVE=$(cat $LOGFILE | grep -wF inactive | awk '{print $2}' | tr -d ',')

From 976ddd39820584fda91023d5147b07aa9c722b66 Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Wed, 3 Apr 2024 10:06:08 -0400
Subject: [PATCH 24/62] add agentstatus to telegraf

---
 salt/telegraf/scripts/agentstatus.sh | 28 ++++++++++++++++------------
 1 file changed, 16 insertions(+), 12 deletions(-)

diff --git a/salt/telegraf/scripts/agentstatus.sh b/salt/telegraf/scripts/agentstatus.sh
index f57fc5f20..a390552fc 100644
--- a/salt/telegraf/scripts/agentstatus.sh
+++ b/salt/telegraf/scripts/agentstatus.sh
@@ -11,19 +11,23 @@
 if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then
 
     LOGFILE=/var/log/agents/agentstatus.log
-    ONLINE=$(cat $LOGFILE | grep -wF online | awk '{print $2}' | tr -d ',')
-    ERROR=$(cat $LOGFILE | grep -wF error | awk '{print $2}' | tr -d ',')
-    INACTIVE=$(cat $LOGFILE | grep -wF inactive | awk '{print $2}' | tr -d ',')
-    OFFLINE=$(cat $LOGFILE | grep -wF offline | awk '{print $2}' | tr -d ',')
-    UPDATING=$(cat $LOGFILE | grep -wF updating | awk '{print $2}' | tr -d ',')
-    UNENROLLED=$(cat $LOGFILE | grep -wF unenrolled | awk '{print $2}' | tr -d ',')
-    OTHER=$(cat $LOGFILE | grep -wF other | awk '{print $2}' | tr -d ',')
-    EVENTS=$(cat $LOGFILE | grep -wF events | awk '{print $2}' | tr -d ',')
-    TOTAL=$(cat $LOGFILE | grep -wF total | awk '{print $2}' | tr -d ',')
-    ALL=$(cat $LOGFILE | grep -wF all | awk '{print $2}' | tr -d ',')
-    ACTIVE=$(cat $LOGFILE | grep -wF active | awk '{print $2}')
 
-    echo "agentstatus online=$ONLINE,error=$ERROR,inactive=$INACTIVE,offline=$OFFLINE,updating=$UPDATING,unenrolled=$UNENROLLED,other=$OTHER,events=$EVENTS,total=$TOTAL,all=$ALL,active=$ACTIVE"
+    # Check to see if the file is there yet so we don't break install verification since there is a 5 minute delay for this file to show up
+    if [ -f $LOGFILE ]; then
+        ONLINE=$(cat $LOGFILE | grep -wF online | awk '{print $2}' | tr -d ',')
+        ERROR=$(cat $LOGFILE | grep -wF error | awk '{print $2}' | tr -d ',')
+        INACTIVE=$(cat $LOGFILE | grep -wF inactive | awk '{print $2}' | tr -d ',')
+        OFFLINE=$(cat $LOGFILE | grep -wF offline | awk '{print $2}' | tr -d ',')
+        UPDATING=$(cat $LOGFILE | grep -wF updating | awk '{print $2}' | tr -d ',')
+        UNENROLLED=$(cat $LOGFILE | grep -wF unenrolled | awk '{print $2}' | tr -d ',')
+        OTHER=$(cat $LOGFILE | grep -wF other | awk '{print $2}' | tr -d ',')
+        EVENTS=$(cat $LOGFILE | grep -wF events | awk '{print $2}' | tr -d ',')
+        TOTAL=$(cat $LOGFILE | grep -wF total | awk '{print $2}' | tr -d ',')
+        ALL=$(cat $LOGFILE | grep -wF all | awk '{print $2}' | tr -d ',')
+        ACTIVE=$(cat $LOGFILE | grep -wF active | awk '{print $2}')
+
+        echo "agentstatus online=$ONLINE,error=$ERROR,inactive=$INACTIVE,offline=$OFFLINE,updating=$UPDATING,unenrolled=$UNENROLLED,other=$OTHER,events=$EVENTS,total=$TOTAL,all=$ALL,active=$ACTIVE"
+    fi
     
 fi
 

From c712529cf6407baf76c7651531e73d30d48c2e5a Mon Sep 17 00:00:00 2001
From: Jason Ertel <jason.ertel@securityonionsolutions.com>
Date: Wed, 3 Apr 2024 10:21:35 -0400
Subject: [PATCH 25/62] suppress soup update output for cleaner console

---
 salt/manager/tools/sbin/soup | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup
index 455fd2256..2191f1ac7 100755
--- a/salt/manager/tools/sbin/soup
+++ b/salt/manager/tools/sbin/soup
@@ -802,7 +802,7 @@ verify_latest_update_script() {
   else
     echo "You are not running the latest soup version. Updating soup and its components. This might take multiple runs to complete."
 
-    salt-call state.apply common.soup_scripts queue=True -linfo --file-root=$UPDATE_DIR/salt --local
+    salt-call state.apply common.soup_scripts queue=True -lerror --file-root=$UPDATE_DIR/salt --local --out-file=/dev/null
 
     # Verify that soup scripts updated as expected
     get_soup_script_hashes

From 5f4a0fdfad79d5815704c1c9d6efbb84d5ccc062 Mon Sep 17 00:00:00 2001
From: Jason Ertel <jason.ertel@securityonionsolutions.com>
Date: Wed, 3 Apr 2024 10:26:48 -0400
Subject: [PATCH 26/62] suppress soup update output for cleaner console

---
 salt/manager/tools/sbin/soup | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup
index 2191f1ac7..cd666fe51 100755
--- a/salt/manager/tools/sbin/soup
+++ b/salt/manager/tools/sbin/soup
@@ -892,17 +892,17 @@ main() {
   echo ""
   require_manager
 
-  check_pillar_items
+  check_pillar_items > /dev/null
 
   echo "Checking to see if this is an airgap install."
   echo ""
-  check_airgap
+  check_airgap > /dev/null
   if [[ $is_airgap -eq 0 && $UNATTENDED == true && -z $ISOLOC ]]; then
     echo "Missing file argument (-f <FILENAME>) for unattended airgap upgrade."
     exit 0
   fi
 
-  set_minionid
+  set_minionid > /dev/null
   echo "Found that Security Onion $INSTALLEDVERSION is currently installed."
   echo ""
   if [[ $is_airgap -eq 0 ]]; then

From 0de1f76139d0b7a26619bca9ae41deace48107e9 Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Wed, 3 Apr 2024 10:26:59 -0400
Subject: [PATCH 27/62] add agent count to reposync

---
 salt/common/tools/sbin/so-common | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common
index 37adcef99..ebff356e5 100755
--- a/salt/common/tools/sbin/so-common
+++ b/salt/common/tools/sbin/so-common
@@ -248,6 +248,11 @@ get_random_value() {
 	head -c 5000 /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w $length | head -n 1
 }
 
+get_agent_count() {
+	
+	AGENTCOUNT=$(/usr/sbin/so-elasticagent-status | grep -wF active | awk '{print $2}')
+}
+
 gpg_rpm_import() {
 	if [[ $is_oracle ]]; then
 	    if [[ "$WHATWOULDYOUSAYYAHDOHERE" == "setup" ]]; then
@@ -570,8 +575,9 @@ sync_options() {
 	set_version
 	set_os
 	salt_minion_count
+	get_agent_count
 	
-	echo "$VERSION/$OS/$(uname -r)/$MINIONCOUNT/$(read_feat)"
+	echo "$VERSION/$OS/$(uname -r)/$MINIONCOUNT/$(read_feat)/$AGENTCOUNT"
 }
 
 systemctl_func() {

From c4767bfdc8ebc610bd70e68838781796e6f300c4 Mon Sep 17 00:00:00 2001
From: Jason Ertel <jason.ertel@securityonionsolutions.com>
Date: Wed, 3 Apr 2024 10:28:43 -0400
Subject: [PATCH 28/62] suppress soup update output for cleaner console

---
 salt/manager/tools/sbin/soup | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup
index cd666fe51..efb8d97d8 100755
--- a/salt/manager/tools/sbin/soup
+++ b/salt/manager/tools/sbin/soup
@@ -229,7 +229,7 @@ check_local_mods() {
 # {% endraw %}
 
 check_pillar_items() {
-  local pillar_output=$(salt-call pillar.items --out=json)
+  local pillar_output=$(salt-call pillar.items -lerror --out=json)
 
   cond=$(jq '.local | has("_errors")' <<< "$pillar_output")
   if [[ "$cond" == "true" ]]; then
@@ -892,7 +892,7 @@ main() {
   echo ""
   require_manager
 
-  check_pillar_items > /dev/null
+  check_pillar_items
 
   echo "Checking to see if this is an airgap install."
   echo ""

From ca57c2069136a4f2abac474bd5c2101b2efda79c Mon Sep 17 00:00:00 2001
From: Jason Ertel <jason.ertel@securityonionsolutions.com>
Date: Wed, 3 Apr 2024 10:31:24 -0400
Subject: [PATCH 29/62] suppress soup update output for cleaner console

---
 salt/common/tools/sbin/so-common | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common
index 37adcef99..e4f2a1e35 100755
--- a/salt/common/tools/sbin/so-common
+++ b/salt/common/tools/sbin/so-common
@@ -329,7 +329,7 @@ lookup_salt_value() {
 		local=""
 	fi
 
-	salt-call --no-color  ${kind}.get ${group}${key} --out=${output} ${local}
+	salt-call -lerror --no-color  ${kind}.get ${group}${key} --out=${output} ${local}
 }
 
 lookup_pillar() {

From 105eadf111f1190eab670dad9b8fa58a0b0f2432 Mon Sep 17 00:00:00 2001
From: Wes <wlambertts@gmail.com>
Date: Wed, 3 Apr 2024 14:40:41 +0000
Subject: [PATCH 30/62] Add cef

---
 salt/elasticsearch/defaults.yaml          | 44 +++++++++++++++++++++++
 salt/elasticsearch/soc_elasticsearch.yaml |  1 +
 2 files changed, 45 insertions(+)

diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml
index c70b0419a..b00a599d8 100644
--- a/salt/elasticsearch/defaults.yaml
+++ b/salt/elasticsearch/defaults.yaml
@@ -2402,6 +2402,50 @@ elasticsearch:
               set_priority:
                 priority: 50
             min_age: 30d
+    so-logs-cef_x_log:
+      index_sorting: False
+      index_template:
+        index_patterns:
+          - "logs-cef.log-*"
+        template:
+          settings:
+            index:
+              lifecycle:
+                name: so-logs-cef.log-logs
+              number_of_replicas: 0
+        composed_of:
+          - "logs-cef.log@package"
+          - "logs-cef.log@custom"
+          - "so-fleet_globals-1"
+          - "so-fleet_agent_id_verification-1"
+        priority: 501
+        data_stream:
+          hidden: false
+          allow_custom_routing: false
+      policy:
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 30d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
     so-logs-checkpoint_x_firewall:
       index_sorting: False
       index_template:
diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml
index e68d0441b..dd82aad6b 100644
--- a/salt/elasticsearch/soc_elasticsearch.yaml
+++ b/salt/elasticsearch/soc_elasticsearch.yaml
@@ -366,6 +366,7 @@ elasticsearch:
     so-logs-azure_x_signinlogs: *indexSettings
     so-logs-azure_x_springcloudlogs: *indexSettings
     so-logs-barracuda_x_waf: *indexSettings
+    so-logs-cef_x_log: *indexSettings
     so-logs-cisco_asa_x_log: *indexSettings
     so-logs-cisco_ftd_x_log: *indexSettings
     so-logs-cisco_ios_x_log: *indexSettings

From c31e459c2b4081436fe8ecad695905f9b90cd044 Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Wed, 3 Apr 2024 11:06:00 -0400
Subject: [PATCH 31/62] Change metrics reporting order

---
 salt/common/tools/sbin/so-common | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common
index ebff356e5..ab3d777ad 100755
--- a/salt/common/tools/sbin/so-common
+++ b/salt/common/tools/sbin/so-common
@@ -249,7 +249,7 @@ get_random_value() {
 }
 
 get_agent_count() {
-	
+
 	AGENTCOUNT=$(/usr/sbin/so-elasticagent-status | grep -wF active | awk '{print $2}')
 }
 
@@ -577,7 +577,7 @@ sync_options() {
 	salt_minion_count
 	get_agent_count
 	
-	echo "$VERSION/$OS/$(uname -r)/$MINIONCOUNT/$(read_feat)/$AGENTCOUNT"
+	echo "$VERSION/$OS/$(uname -r)/$MINIONCOUNT:$AGENTCOUNT/$(read_feat)"
 }
 
 systemctl_func() {

From 66844af1c2a6c5e5baee0cefd2d60a87907fee82 Mon Sep 17 00:00:00 2001
From: Doug Burks <doug.burks@securityonionsolutions.com>
Date: Wed, 3 Apr 2024 11:54:53 -0400
Subject: [PATCH 32/62] FEATURE: Add dashboard for SOC Login Failures #12738

---
 salt/soc/defaults.yaml | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml
index 21b107367..e6fbb742a 100644
--- a/salt/soc/defaults.yaml
+++ b/salt/soc/defaults.yaml
@@ -1656,9 +1656,12 @@ soc:
             - name: Overview
               description: Overview of all events
               query: '* | groupby event.category | groupby -sankey event.category event.module | groupby event.module | groupby -sankey event.module event.dataset | groupby event.dataset | groupby observer.name | groupby host.name | groupby source.ip | groupby destination.ip | groupby destination.port'
-            - name: SOC Auth
-              description: SOC (Security Onion Console) authentication logs
+            - name: SOC Logins
+              description: SOC (Security Onion Console) logins
               query: 'event.dataset:kratos.audit AND msg:*authenticated* | groupby http_request.headers.x-real-ip | groupby -sankey http_request.headers.x-real-ip identity_id | groupby identity_id | groupby http_request.headers.user-agent'
+            - name: SOC Login Failures
+              description: SOC (Security Onion Console) login failures
+              query: 'event.dataset:kratos.audit AND msg:*Encountered*self-service*login*error* | groupby http_request.headers.x-real-ip | groupby -sankey http_request.headers.x-real-ip http_request.headers.user-agent | groupby http_request.headers.user-agent'
             - name: Elastalerts
               description: Elastalert logs
               query: '_index: "*:elastalert*" | groupby rule_name | groupby alert_info.type'

From 8889c974b85e42f47706d6ec37aa2b3cef7935cb Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Wed, 3 Apr 2024 12:38:59 -0400
Subject: [PATCH 33/62] Change code to allow for non root

---
 salt/common/tools/sbin/so-common | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common
index ab3d777ad..9078826e5 100755
--- a/salt/common/tools/sbin/so-common
+++ b/salt/common/tools/sbin/so-common
@@ -249,8 +249,11 @@ get_random_value() {
 }
 
 get_agent_count() {
-
-	AGENTCOUNT=$(/usr/sbin/so-elasticagent-status | grep -wF active | awk '{print $2}')
+	if [ -f /opt/so/log/agents/agentstatus.log ]; then
+		AGENTCOUNT=$(cat /opt/so/log/agents/agentstatus.log | grep -wF active | awk '{print $2}')
+	else
+		AGENTCOUNT=0
+	fi
 }
 
 gpg_rpm_import() {

From 9078b2bad29b172708083eafbd4870d13a144d7d Mon Sep 17 00:00:00 2001
From: Doug Burks <doug.burks@securityonionsolutions.com>
Date: Wed, 3 Apr 2024 12:46:29 -0400
Subject: [PATCH 34/62] FEATURE: Add Events table columns for event.module
 kratos #12740

---
 salt/soc/defaults.yaml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml
index e6fbb742a..3e757e431 100644
--- a/salt/soc/defaults.yaml
+++ b/salt/soc/defaults.yaml
@@ -87,12 +87,13 @@ soc:
         - log.id.uid
         - network.community_id
         - event.dataset
-      ':kratos:audit':
+      ':kratos:':
         - soc_timestamp
         - http_request.headers.x-real-ip
         - identity_id
         - http_request.headers.user-agent
         - event.dataset
+        - msg
       '::conn':
         - soc_timestamp
         - source.ip

From 2b8a051525b7157c005741d56f565f6cd5f36726 Mon Sep 17 00:00:00 2001
From: Jason Ertel <jason.ertel@securityonionsolutions.com>
Date: Wed, 3 Apr 2024 14:30:09 -0400
Subject: [PATCH 35/62] fix link

---
 salt/manager/tools/sbin/soup | 6 +++---
 setup/so-whiptail            | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup
index efb8d97d8..58e6bd006 100755
--- a/salt/manager/tools/sbin/soup
+++ b/salt/manager/tools/sbin/soup
@@ -598,7 +598,7 @@ used and enables informed prioritization of future development.
 
 Adjust this setting at anytime via the SOC Configuration screen.
 
-Additional information: https://docs.securityonion.net/telemetry.rst.
+Additional information: https://docs.securityonion.net/en/2.4/telemetry.html
 
 ASSIST_EOF
 
@@ -896,13 +896,13 @@ main() {
 
   echo "Checking to see if this is an airgap install."
   echo ""
-  check_airgap > /dev/null
+  check_airgap
   if [[ $is_airgap -eq 0 && $UNATTENDED == true && -z $ISOLOC ]]; then
     echo "Missing file argument (-f <FILENAME>) for unattended airgap upgrade."
     exit 0
   fi
 
-  set_minionid > /dev/null
+  set_minionid
   echo "Found that Security Onion $INSTALLEDVERSION is currently installed."
   echo ""
   if [[ $is_airgap -eq 0 ]]; then
diff --git a/setup/so-whiptail b/setup/so-whiptail
index 5fa5bf343..e79f35d2f 100755
--- a/setup/so-whiptail
+++ b/setup/so-whiptail
@@ -156,7 +156,7 @@ whiptail_accept_telemetry() {
 
 	Adjust this setting at anytime via the SOC Configuration screen.
 
-	Additional information: https://docs.securityonion.net/telemetry.rst.
+	Additional information: https://docs.securityonion.net/en/2.4/telemetry.html
 
 	Enable SOC Telemetry to help improve future releases?
 	EOM

From 941a841da0f9f4d5ce8256acb7513ffb09143922 Mon Sep 17 00:00:00 2001
From: Jason Ertel <jason.ertel@securityonionsolutions.com>
Date: Wed, 3 Apr 2024 14:41:57 -0400
Subject: [PATCH 36/62] fix link

---
 salt/manager/tools/sbin/soup | 2 +-
 setup/so-whiptail            | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup
index 58e6bd006..bf2e74bf7 100755
--- a/salt/manager/tools/sbin/soup
+++ b/salt/manager/tools/sbin/soup
@@ -598,7 +598,7 @@ used and enables informed prioritization of future development.
 
 Adjust this setting at anytime via the SOC Configuration screen.
 
-Additional information: https://docs.securityonion.net/en/2.4/telemetry.html
+Read more: https://docs.securityonion.net/en/2.4/telemetry.html
 
 ASSIST_EOF
 
diff --git a/setup/so-whiptail b/setup/so-whiptail
index e79f35d2f..de1cfec83 100755
--- a/setup/so-whiptail
+++ b/setup/so-whiptail
@@ -156,7 +156,7 @@ whiptail_accept_telemetry() {
 
 	Adjust this setting at anytime via the SOC Configuration screen.
 
-	Additional information: https://docs.securityonion.net/en/2.4/telemetry.html
+	Documentation: https://docs.securityonion.net/en/2.4/telemetry.html
 
 	Enable SOC Telemetry to help improve future releases?
 	EOM

From 8f8896c505d43092c5ef223278f8d3f11ba28363 Mon Sep 17 00:00:00 2001
From: Jason Ertel <jason.ertel@securityonionsolutions.com>
Date: Wed, 3 Apr 2024 14:45:39 -0400
Subject: [PATCH 37/62] fix link

---
 salt/manager/tools/sbin/soup | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup
index bf2e74bf7..99191a442 100755
--- a/salt/manager/tools/sbin/soup
+++ b/salt/manager/tools/sbin/soup
@@ -598,7 +598,7 @@ used and enables informed prioritization of future development.
 
 Adjust this setting at anytime via the SOC Configuration screen.
 
-Read more: https://docs.securityonion.net/en/2.4/telemetry.html
+Documentation: https://docs.securityonion.net/en/2.4/telemetry.html
 
 ASSIST_EOF
 

From 0f50a265cf2ab21dd1eee2ca846d64c0745622c6 Mon Sep 17 00:00:00 2001
From: Corey Ogburn <corey.ogburn@securityonionsolutions.com>
Date: Wed, 3 Apr 2024 13:12:18 -0600
Subject: [PATCH 38/62] Update SOC Config with State File Paths

Each detection engine is getting a state file to help manage the timer over restarts. By default, the files will go in soc's config folder inside a fingerprints folder.
---
 salt/soc/defaults.yaml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml
index 4ac77229e..0c9d8506e 100644
--- a/salt/soc/defaults.yaml
+++ b/salt/soc/defaults.yaml
@@ -1247,6 +1247,7 @@ soc:
           elastAlertRulesFolder: /opt/sensoroni/elastalert
           reposFolder: /opt/sensoroni/sigma/repos
           rulesFingerprintFile: /opt/sensoroni/fingerprints/sigma.fingerprint
+          stateFilePath: /opt/so/conf/soc/fingerprints/elastalertengine.state
           rulesRepos:
           - repo: https://github.com/Security-Onion-Solutions/securityonion-resources
             license: Elastic-2.0
@@ -1307,6 +1308,7 @@ soc:
           - repo: https://github.com/Security-Onion-Solutions/securityonion-yara
             license: DRL
           yaraRulesFolder: /opt/sensoroni/yara/rules
+          stateFilePath: /opt/so/conf/soc/fingerprints/strelkaengine.state
         suricataengine:
           allowRegex: ''
           autoUpdateEnabled: true
@@ -1314,6 +1316,7 @@ soc:
           communityRulesFile: /nsm/rules/suricata/emerging-all.rules
           denyRegex: ''
           rulesFingerprintFile: /opt/sensoroni/fingerprints/emerging-all.fingerprint
+          stateFilePath: /opt/so/conf/soc/fingerprints/suricataengine.state
       client:
         enableReverseLookup: false
         docsUrl: /docs/

From 9db9af27aed759eaceeaf0132608c8fc4e2d0c97 Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Wed, 3 Apr 2024 15:14:50 -0400
Subject: [PATCH 39/62] Attempt to fix 2.3 when main repo changes

---
 salt/manager/tools/sbin/soup | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup
index a585f877c..9e972b4bc 100755
--- a/salt/manager/tools/sbin/soup
+++ b/salt/manager/tools/sbin/soup
@@ -837,7 +837,16 @@ main() {
 
   echo "### Preparing soup at $(date) ###"
   echo ""
-
+  if [[ "$INSTALLEDVERISON" =~ ^2.3.* ]]; then
+    BRANCH="2.3/main"
+    cd $UPDATE_DIR
+    clone_to_tmp
+    cp $UPDATE_DIR/salt/common/tools/sbin/soup /usr/sbin
+    cp $UPDATE_DIR/salt/common/tools/sbin/soup /opt/so/saltstack/default/salt/common/tools/sbin
+    add_common
+    echo "Please run soup again"
+    exit 0
+  fi
   set_os
 
   check_salt_master_status 1 || fail  "Could not talk to salt master: Please run 'systemctl status salt-master' to ensure the salt-master service is running and check the log at /opt/so/log/salt/master."

From 9c59f42c16a9232abf1e6de8f03c941f617a46bd Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Wed, 3 Apr 2024 15:23:09 -0400
Subject: [PATCH 40/62] Attempt to fix 2.3 when main repo changes

---
 salt/manager/tools/sbin/soup | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup
index 9e972b4bc..fd5ee8ead 100755
--- a/salt/manager/tools/sbin/soup
+++ b/salt/manager/tools/sbin/soup
@@ -841,8 +841,8 @@ main() {
     BRANCH="2.3/main"
     cd $UPDATE_DIR
     clone_to_tmp
-    cp $UPDATE_DIR/salt/common/tools/sbin/soup /usr/sbin
-    cp $UPDATE_DIR/salt/common/tools/sbin/soup /opt/so/saltstack/default/salt/common/tools/sbin
+    cp $UPDATE_DIR/securityonion/salt/common/tools/sbin/soup /usr/sbin
+    cp $UPDATE_DIR/securityonion/salt/common/tools/sbin/soup /opt/so/saltstack/default/salt/common/tools/sbin
     add_common
     echo "Please run soup again"
     exit 0

From 12da7db22c57744a73d48b6028823c98491c5ec0 Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Wed, 3 Apr 2024 15:38:23 -0400
Subject: [PATCH 41/62] Attempt to fix 2.3 when main repo changes

---
 salt/manager/tools/sbin/soup | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup
index fd5ee8ead..cbe3d461a 100755
--- a/salt/manager/tools/sbin/soup
+++ b/salt/manager/tools/sbin/soup
@@ -837,7 +837,8 @@ main() {
 
   echo "### Preparing soup at $(date) ###"
   echo ""
-  if [[ "$INSTALLEDVERISON" =~ ^2.3.* ]]; then
+  if [[ "$INSTALLEDVERSION" =~ ^2.3 ]]; then
+    echo "Actually running the code"
     BRANCH="2.3/main"
     cd $UPDATE_DIR
     clone_to_tmp

From f66cca96ce935cd6ff1e95a5516514527018c387 Mon Sep 17 00:00:00 2001
From: DefensiveDepth <Josh@defensivedepth.com>
Date: Wed, 3 Apr 2024 16:17:29 -0400
Subject: [PATCH 42/62] YARA casing

---
 salt/soc/defaults.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml
index 1d0eb0e38..9bb302057 100644
--- a/salt/soc/defaults.yaml
+++ b/salt/soc/defaults.yaml
@@ -2053,7 +2053,7 @@ soc:
               query: "so_detection.language:sigma"
             - name: "Detection Type - Sigma - Windows"
               query: 'so_detection.language:sigma AND so_detection.content: "*product: windows*"'
-            - name: "Detection Type - Yara (Strelka)"
+            - name: "Detection Type - YARA (Strelka)"
               query: "so_detection.language:yara"
             - name: "Security Onion - Grid Detections"
               query: "so_detection.ruleset:securityonion-resources"

From 7c64fc8c05fc8e16592ae4344f49226acc7145bc Mon Sep 17 00:00:00 2001
From: Jason Ertel <jason.ertel@securityonionsolutions.com>
Date: Wed, 3 Apr 2024 18:08:42 -0400
Subject: [PATCH 43/62] do not prompt about telemetry on airgap installs

---
 setup/so-setup | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/setup/so-setup b/setup/so-setup
index fc13e5b18..1c3be22bf 100755
--- a/setup/so-setup
+++ b/setup/so-setup
@@ -447,7 +447,7 @@ if ! [[ -f $install_opt_file ]]; then
 		get_redirect
 		# Does the user want to allow access to the UI?
 		collect_so_allow
-		whiptail_accept_telemetry
+		[[ ! $is_airgap ]] && whiptail_accept_telemetry
 		whiptail_end_settings
 	elif [[ $is_standalone ]]; then
 		waitforstate=true
@@ -469,7 +469,7 @@ if ! [[ -f $install_opt_file ]]; then
 		collect_webuser_inputs
 		get_redirect
 		collect_so_allow
-		whiptail_accept_telemetry
+		[[ ! $is_airgap ]] && whiptail_accept_telemetry
 		whiptail_end_settings
 	elif [[ $is_manager ]]; then
 		info "Setting up as node type manager"
@@ -490,7 +490,7 @@ if ! [[ -f $install_opt_file ]]; then
 		collect_webuser_inputs
 		get_redirect
 		collect_so_allow
-		whiptail_accept_telemetry
+		[[ ! $is_airgap ]] && whiptail_accept_telemetry
 		whiptail_end_settings
 	elif [[ $is_managersearch ]]; then
 		info "Setting up as node type managersearch"
@@ -511,7 +511,7 @@ if ! [[ -f $install_opt_file ]]; then
 		collect_webuser_inputs
 		get_redirect
 		collect_so_allow
-		whiptail_accept_telemetry
+		[[ ! $is_airgap ]] && whiptail_accept_telemetry
 		whiptail_end_settings
 	elif [[ $is_sensor ]]; then
 		info "Setting up as node type sensor"
@@ -601,7 +601,7 @@ if ! [[ -f $install_opt_file ]]; then
 		collect_webuser_inputs
 		get_redirect
 		collect_so_allow
-		whiptail_accept_telemetry
+		[[ ! $is_airgap ]] && whiptail_accept_telemetry
 		whiptail_end_settings
 
 	elif [[ $is_receiver ]]; then

From 1d221a574b9150915de87aa0700dc36a687d5ba9 Mon Sep 17 00:00:00 2001
From: DefensiveDepth <Josh@defensivedepth.com>
Date: Thu, 4 Apr 2024 06:48:25 -0400
Subject: [PATCH 44/62] Exclude Elastalert EQL errors

---
 salt/common/tools/sbin/so-log-check | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check
index a4b25f0f3..cc5fef85b 100755
--- a/salt/common/tools/sbin/so-log-check
+++ b/salt/common/tools/sbin/so-log-check
@@ -198,6 +198,8 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|req.LocalMeta.host.ip"        # known issue in GH
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|sendmail"                     # zeek
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|stats.log"
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Unknown column"               # Elastalert errors from running EQL queries
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|parsing_exception"            # Elastalert EQL parsing issue. Temp.
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|context deadline exceeded"
 fi
 

From a9517e1291b6fc67c5805c066e2092236583b8f2 Mon Sep 17 00:00:00 2001
From: Jason Ertel <jason.ertel@securityonionsolutions.com>
Date: Thu, 4 Apr 2024 07:49:30 -0400
Subject: [PATCH 45/62] clarify telemetry annotation

---
 salt/soc/soc_soc.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml
index eed0113fc..c013b66e8 100644
--- a/salt/soc/soc_soc.yaml
+++ b/salt/soc/soc_soc.yaml
@@ -4,7 +4,7 @@ soc:
     advanced: True
   telemetryEnabled:
     title: SOC Telemetry
-    description: When enabled, SOC provides feature usage data to the Security Onion development team via Google Analytics. This data helps Security Onion developers determine which product features are being used and can also provide insight into improving the user interface. When changing this setting, wait for the grid to fully synchronize and then perform a hard browser refresh on SOC, to force the browser cache to update and reflect the new setting.
+    description: When enabled and not in airgap mode, SOC provides feature usage data to the Security Onion development team via Google Analytics. This data helps Security Onion developers determine which product features are being used and can also provide insight into improving the user interface. When changing this setting, wait for the grid to fully synchronize and then perform a hard browser refresh on SOC, to force the browser cache to update and reflect the new setting.
     global: True
     helpLink: telemetry.html
   files:

From a7fab380b40ba2df587a97e12f470b66a48b55d8 Mon Sep 17 00:00:00 2001
From: Jason Ertel <jason.ertel@securityonionsolutions.com>
Date: Thu, 4 Apr 2024 07:51:23 -0400
Subject: [PATCH 46/62] clarify telemetry annotation

---
 salt/soc/soc_soc.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml
index c013b66e8..a9f0db6c2 100644
--- a/salt/soc/soc_soc.yaml
+++ b/salt/soc/soc_soc.yaml
@@ -4,7 +4,7 @@ soc:
     advanced: True
   telemetryEnabled:
     title: SOC Telemetry
-    description: When enabled and not in airgap mode, SOC provides feature usage data to the Security Onion development team via Google Analytics. This data helps Security Onion developers determine which product features are being used and can also provide insight into improving the user interface. When changing this setting, wait for the grid to fully synchronize and then perform a hard browser refresh on SOC, to force the browser cache to update and reflect the new setting.
+    description: When this setting is enabled and the grid is not in airgap mode, SOC will provide feature usage data to the Security Onion development team via Google Analytics. This data helps Security Onion developers determine which product features are being used and can also provide insight into improving the user interface. When changing this setting, wait for the grid to fully synchronize and then perform a hard browser refresh on SOC, to force the browser cache to update and reflect the new setting.
     global: True
     helpLink: telemetry.html
   files:

From 14c824143bfb821f1b553fa24d68605c15461556 Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Thu, 4 Apr 2024 08:48:44 -0400
Subject: [PATCH 47/62] Attempt to fix 2.3 when main repo changes

---
 salt/common/soup_scripts.sls |  7 +++++++
 salt/manager/tools/sbin/soup | 11 -----------
 2 files changed, 7 insertions(+), 11 deletions(-)

diff --git a/salt/common/soup_scripts.sls b/salt/common/soup_scripts.sls
index fd32b8a28..99358a1a1 100644
--- a/salt/common/soup_scripts.sls
+++ b/salt/common/soup_scripts.sls
@@ -1,3 +1,5 @@
+{% if '2.4' in salt['cp.get_file_str']('/etc/soversion') %}
+
 {% import_yaml '/opt/so/saltstack/local/pillar/global/soc_global.sls' as SOC_GLOBAL %}
 {% if SOC_GLOBAL.global.airgap %}
 {%   set UPDATE_DIR='/tmp/soagupdate/SecurityOnion' %}
@@ -68,3 +70,8 @@ copy_so-firewall_sbin:
     - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-firewall
     - force: True
     - preserve: True
+{% else %}
+fix_old_versions:
+  cmd.run:
+    - name: BRANCH=2.3/main soup -y
+{% endif %}
\ No newline at end of file
diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup
index cbe3d461a..a4343faf4 100755
--- a/salt/manager/tools/sbin/soup
+++ b/salt/manager/tools/sbin/soup
@@ -837,17 +837,6 @@ main() {
 
   echo "### Preparing soup at $(date) ###"
   echo ""
-  if [[ "$INSTALLEDVERSION" =~ ^2.3 ]]; then
-    echo "Actually running the code"
-    BRANCH="2.3/main"
-    cd $UPDATE_DIR
-    clone_to_tmp
-    cp $UPDATE_DIR/securityonion/salt/common/tools/sbin/soup /usr/sbin
-    cp $UPDATE_DIR/securityonion/salt/common/tools/sbin/soup /opt/so/saltstack/default/salt/common/tools/sbin
-    add_common
-    echo "Please run soup again"
-    exit 0
-  fi
   set_os
 
   check_salt_master_status 1 || fail  "Could not talk to salt master: Please run 'systemctl status salt-master' to ensure the salt-master service is running and check the log at /opt/so/log/salt/master."

From c2f7f7e3a5dbd562c28691722ef4ff07858a2954 Mon Sep 17 00:00:00 2001
From: DefensiveDepth <Josh@defensivedepth.com>
Date: Thu, 4 Apr 2024 08:52:30 -0400
Subject: [PATCH 48/62] Remove dup line

---
 salt/manager/tools/sbin/soup | 1 -
 1 file changed, 1 deletion(-)

diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup
index 176c290a8..dba3215d1 100755
--- a/salt/manager/tools/sbin/soup
+++ b/salt/manager/tools/sbin/soup
@@ -375,7 +375,6 @@ postupgrade_changes() {
     [[ "$POSTVERSION"  == 2.4.40 ]] && post_to_2.4.50
     [[ "$POSTVERSION"  == 2.4.50 ]] && post_to_2.4.60
     [[ "$POSTVERSION"  == 2.4.60 ]] && post_to_2.4.70
-    [[ "$POSTVERSION"  == 2.4.60 ]] && post_to_2.4.70
     true
 }
 

From 4b31632dfc3764773763c2808ccec802520eeee6 Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Thu, 4 Apr 2024 08:52:37 -0400
Subject: [PATCH 49/62] Attempt to fix 2.3 when main repo changes

---
 salt/common/soup_scripts.sls | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/salt/common/soup_scripts.sls b/salt/common/soup_scripts.sls
index 99358a1a1..da1f5199b 100644
--- a/salt/common/soup_scripts.sls
+++ b/salt/common/soup_scripts.sls
@@ -73,5 +73,6 @@ copy_so-firewall_sbin:
 {% else %}
 fix_old_versions:
   cmd.run:
-    - name: BRANCH=2.3/main soup -y
+    - env: BRANCH=2.3/main
+    - name: soup -y
 {% endif %}
\ No newline at end of file

From d3f163bf9e1e46c9baa075a8cbaa73479edb9aa4 Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Thu, 4 Apr 2024 08:54:04 -0400
Subject: [PATCH 50/62] Attempt to fix 2.3 when main repo changes

---
 salt/common/soup_scripts.sls | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/common/soup_scripts.sls b/salt/common/soup_scripts.sls
index da1f5199b..c8489ac64 100644
--- a/salt/common/soup_scripts.sls
+++ b/salt/common/soup_scripts.sls
@@ -73,6 +73,6 @@ copy_so-firewall_sbin:
 {% else %}
 fix_old_versions:
   cmd.run:
-    - env: BRANCH=2.3/main
+    - env: "BRANCH=2.3/main"
     - name: soup -y
 {% endif %}
\ No newline at end of file

From 470b0e4bf68415835f2b2c0f5e2b311ba120425c Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Thu, 4 Apr 2024 08:55:13 -0400
Subject: [PATCH 51/62] Attempt to fix 2.3 when main repo changes

---
 salt/common/soup_scripts.sls | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/salt/common/soup_scripts.sls b/salt/common/soup_scripts.sls
index c8489ac64..fe378c38f 100644
--- a/salt/common/soup_scripts.sls
+++ b/salt/common/soup_scripts.sls
@@ -73,6 +73,7 @@ copy_so-firewall_sbin:
 {% else %}
 fix_old_versions:
   cmd.run:
-    - env: "BRANCH=2.3/main"
+    - env: 
+      - BRANCH: "2.3/main"
     - name: soup -y
 {% endif %}
\ No newline at end of file

From 7668fa1396dce0757e370d6e8ef0448a0f18a13c Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Thu, 4 Apr 2024 09:03:29 -0400
Subject: [PATCH 52/62] Attempt to fix 2.3 when main repo changes

---
 salt/common/soup_scripts.sls | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/salt/common/soup_scripts.sls b/salt/common/soup_scripts.sls
index fe378c38f..f36ddf47b 100644
--- a/salt/common/soup_scripts.sls
+++ b/salt/common/soup_scripts.sls
@@ -71,9 +71,10 @@ copy_so-firewall_sbin:
     - force: True
     - preserve: True
 {% else %}
-fix_old_versions:
+fix_23_soup_sbin:
   cmd.run:
-    - env: 
-      - BRANCH: "2.3/main"
-    - name: soup -y
+    - name: curl -s -o /usr/sbin/soup https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.3/main/salt/common/tools/sbin/soup
+fix_23_soup_salt:
+  cmd.run:
+    - name: curl -s -o /opt/so/saltstack/defalt/salt/common/tools/sbin/soup https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.3/main/salt/common/tools/sbin/soup
 {% endif %}
\ No newline at end of file

From 5ec3b834fb16d0ac862d40589ebf4bf3d667fbaa Mon Sep 17 00:00:00 2001
From: Doug Burks <doug.burks@securityonionsolutions.com>
Date: Thu, 4 Apr 2024 09:11:41 -0400
Subject: [PATCH 53/62] FEATURE: Add Events table columns for event.module
 sigma #12743

---
 salt/soc/defaults.yaml | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml
index 0c9d8506e..17d4464d4 100644
--- a/salt/soc/defaults.yaml
+++ b/salt/soc/defaults.yaml
@@ -1222,6 +1222,17 @@ soc:
         - event_data.destination.port
         - event_data.process.executable
         - event_data.process.pid
+      ':sigma:':
+        - soc_timestamp
+        - rule.name
+        - event.severity_label
+        - event_data.event.dataset
+        - event_data.source.ip
+        - event_data.source.port
+        - event_data.destination.host
+        - event_data.destination.port
+        - event_data.process.executable
+        - event_data.process.pid
     server:
       bindAddress: 0.0.0.0:9822
       baseUrl: /
@@ -1915,6 +1926,17 @@ soc:
               - event_data.destination.port
               - event_data.process.executable
               - event_data.process.pid    
+            ':sigma:':
+              - soc_timestamp
+              - rule.name
+              - event.severity_label
+              - event_data.event.dataset
+              - event_data.source.ip
+              - event_data.source.port
+              - event_data.destination.host
+              - event_data.destination.port
+              - event_data.process.executable
+              - event_data.process.pid    
             ':strelka:':
               - soc_timestamp
               - file.name

From c4ebe25bab42761482c63461344cee10ae35f703 Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Thu, 4 Apr 2024 09:18:37 -0400
Subject: [PATCH 54/62] Attempt to fix 2.3 when main repo changes

---
 salt/common/soup_scripts.sls | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/salt/common/soup_scripts.sls b/salt/common/soup_scripts.sls
index f36ddf47b..e86edfad1 100644
--- a/salt/common/soup_scripts.sls
+++ b/salt/common/soup_scripts.sls
@@ -73,8 +73,8 @@ copy_so-firewall_sbin:
 {% else %}
 fix_23_soup_sbin:
   cmd.run:
-    - name: curl -s -o /usr/sbin/soup https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.3/main/salt/common/tools/sbin/soup
+    - name: curl -s -f -o /usr/sbin/soup https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.3/main/salt/common/tools/sbin/soup
 fix_23_soup_salt:
   cmd.run:
-    - name: curl -s -o /opt/so/saltstack/defalt/salt/common/tools/sbin/soup https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.3/main/salt/common/tools/sbin/soup
+    - name: curl -s -f -o /opt/so/saltstack/defalt/salt/common/tools/sbin/soup https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.3/main/salt/common/tools/sbin/soup
 {% endif %}
\ No newline at end of file

From 784ec54795008dd2f37aaedee364fc946e3de586 Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Thu, 4 Apr 2024 09:24:17 -0400
Subject: [PATCH 55/62] 2.3 updates

---
 salt/common/soup_scripts.sls | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/salt/common/soup_scripts.sls b/salt/common/soup_scripts.sls
index d0f8589a0..90ee059a4 100644
--- a/salt/common/soup_scripts.sls
+++ b/salt/common/soup_scripts.sls
@@ -1,11 +1,11 @@
 {% if '2.4' in salt['cp.get_file_str']('/etc/soversion') %}
 
-{% import_yaml '/opt/so/saltstack/local/pillar/global/soc_global.sls' as SOC_GLOBAL %}
-{% if SOC_GLOBAL.global.airgap %}
-{%   set UPDATE_DIR='/tmp/soagupdate/SecurityOnion' %}
-{% else %}
-{%   set UPDATE_DIR='/tmp/sogh/securityonion' %}
-{% endif %}
+{%   import_yaml '/opt/so/saltstack/local/pillar/global/soc_global.sls' as SOC_GLOBAL %}
+{%   if SOC_GLOBAL.global.airgap %}
+{%     set UPDATE_DIR='/tmp/soagupdate/SecurityOnion' %}
+{%   else %}
+{%     set UPDATE_DIR='/tmp/sogh/securityonion' %}
+{%   endif %}
 
 remove_common_soup:
   file.absent:

From d8ac3f1292b0ffa2aaec798e4dbb9d19eb6f29dd Mon Sep 17 00:00:00 2001
From: Doug Burks <doug.burks@securityonionsolutions.com>
Date: Thu, 4 Apr 2024 09:30:05 -0400
Subject: [PATCH 56/62] FEATURE: Add dashboards specific to Elastic Agent
 #12746

---
 salt/soc/defaults.yaml | 48 ++++++++++++++++++++++++++----------------
 1 file changed, 30 insertions(+), 18 deletions(-)

diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml
index 17d4464d4..7c5c5b044 100644
--- a/salt/soc/defaults.yaml
+++ b/salt/soc/defaults.yaml
@@ -1678,18 +1678,36 @@ soc:
             - name: SOC Login Failures
               description: SOC (Security Onion Console) login failures
               query: 'event.dataset:kratos.audit AND msg:*Encountered*self-service*login*error* | groupby http_request.headers.x-real-ip | groupby -sankey http_request.headers.x-real-ip http_request.headers.user-agent | groupby http_request.headers.user-agent'
-            - name: Elastalerts
-              description: Elastalert logs
-              query: '_index: "*:elastalert*" | groupby rule_name | groupby alert_info.type'
             - name: Alerts
               description: Overview of all alerts
               query: 'tags:alert | groupby event.module* | groupby source.ip | groupby -sankey source.ip destination.ip | groupby destination.ip | groupby destination.port | groupby rule.name | groupby event.severity | groupby destination_geo.organization_name'
             - name: NIDS Alerts
               description: NIDS (Network Intrusion Detection System) alerts
               query: 'event.category:network AND tags:alert | groupby rule.category | groupby source.ip | groupby -sankey source.ip destination.ip | groupby destination.ip | groupby destination.port | groupby rule.name | groupby rule.uuid | groupby rule.gid | groupby destination_geo.organization_name'
-            - name: Sysmon Overview
-              description: Overview of all Sysmon data types
-              query: 'event.dataset:windows.sysmon_operational | groupby event.action | groupby -sankey event.action host.name | groupby host.name | groupby -sankey host.name user.name | groupby user.name | groupby event.category event.action | groupby dns.question.name | groupby process.executable | groupby file.name | groupby source.ip | groupby destination.ip | groupby destination.port'
+            - name: Elastic Agent Overview
+              description: Overview of all events from Elastic Agents
+              query: 'event.module:endpoint | groupby event.dataset | groupby host.name | groupby -sankey host.name user.name | groupby user.name | groupby -sankey user.name process.name | groupby process.name'
+            - name: Elastic Agent API Events
+              description: API (Application Programming Interface) events from Elastic Agents
+              query: 'event.dataset:endpoint.events.api | groupby host.name | groupby -sankey host.name user.name | groupby user.name | groupby -sankey user.name process.name | groupby process.name | groupby process.Ext.api.name'
+            - name: Elastic Agent File Events
+              description: File events from Elastic Agents
+              query: 'event.dataset:endpoint.events.file | groupby host.name | groupby -sankey host.name user.name | groupby user.name | groupby -sankey user.name process.name | groupby process.name | groupby event.action | groupby file.path'
+            - name: Elastic Agent Library Events
+              description: Library events from Elastic Agents
+              query: 'event.dataset:endpoint.events.library | groupby host.name | groupby -sankey host.name user.name | groupby user.name | groupby -sankey user.name process.name | groupby process.name | groupby event.action | groupby dll.path | groupby dll.code_signature.status | groupby dll.code_signature.subject_name'
+            - name: Elastic Agent Network Events
+              description: Network events from Elastic Agents
+              query: 'event.dataset:endpoint.events.network | groupby host.name | groupby -sankey host.name user.name | groupby user.name | groupby -sankey user.name process.name | groupby process.name | groupby event.action | groupby source.ip | groupby destination.ip | groupby destination.port'
+            - name: Elastic Agent Process Events
+              description: Process events from Elastic Agents
+              query: 'event.dataset:endpoint.events.process | groupby host.name | groupby -sankey host.name user.name | groupby user.name | groupby -sankey user.name process.parent.name | groupby process.parent.name | groupby -sankey process.parent.name process.name | groupby process.name | groupby event.action | groupby process.working_directory'
+            - name: Elastic Agent Registry Events
+              description: Registry events from Elastic Agents
+              query: 'event.dataset:endpoint.events.registry | groupby host.name | groupby -sankey host.name user.name | groupby user.name | groupby -sankey user.name process.name | groupby process.name | groupby event.action | groupby registry.path'
+            - name: Elastic Agent Security Events
+              description: Security events from Elastic Agents
+              query: 'event.dataset:endpoint.events.security | groupby host.name | groupby -sankey host.name user.name | groupby user.name | groupby -sankey user.name process.executable | groupby process.executable | groupby event.action | groupby event.outcome'
             - name: Host Overview
               description: Overview of all host data types
               query: '((event.category:registry OR event.category:host OR event.category:process OR event.category:driver OR event.category:configuration) OR (event.category:file  AND _exists_:process.executable) OR (event.category:network AND _exists_:host.name)) | groupby event.dataset* event.category* event.action* | groupby event.type | groupby -sankey event.type host.name | groupby host.name | groupby user.name | groupby file.name | groupby process.executable'
@@ -1708,24 +1726,18 @@ soc:
             - name: Host Network & Process Mappings
               description: Network activity mapped to originating processes
               query: 'event.category: network AND _exists_:process.executable | groupby event.action | groupby -sankey event.action host.name | groupby host.name | groupby -sankey host.name user.name | groupby user.name | groupby event.dataset* event.type* event.action* | groupby dns.question.name | groupby process.executable | groupby process.name | groupby source.ip | groupby destination.ip | groupby destination.port'
-            - name: Host API Events
-              description: API (Application Programming Interface) events from endpoints
-              query: 'event.dataset:endpoint.events.api | groupby host.name | groupby -sankey host.name user.name | groupby user.name | groupby -sankey user.name process.name | groupby process.name | groupby process.Ext.api.name'
-            - name: Host Library Events
-              description: Library events from endpoints
-              query: 'event.dataset:endpoint.events.library | groupby host.name | groupby -sankey host.name user.name | groupby user.name | groupby -sankey user.name process.name | groupby process.name | groupby event.action | groupby dll.path | groupby dll.code_signature.status | groupby dll.code_signature.subject_name'
-            - name: Host Security Events
-              description: Security events from endpoints
-              query: 'event.dataset:endpoint.events.security | groupby host.name | groupby -sankey host.name user.name | groupby user.name | groupby -sankey user.name process.executable | groupby process.executable | groupby event.action | groupby event.outcome'
+            - name: Sysmon Overview
+              description: Overview of all Sysmon data types
+              query: 'event.dataset:windows.sysmon_operational | groupby event.action | groupby -sankey event.action host.name | groupby host.name | groupby -sankey host.name user.name | groupby user.name | groupby event.category event.action | groupby dns.question.name | groupby process.executable | groupby file.name | groupby source.ip | groupby destination.ip | groupby destination.port'
             - name: Strelka
               description: Strelka file analysis
               query: 'event.module:strelka | groupby file.mime_type | groupby -sankey file.mime_type file.source | groupby file.source | groupby -sankey file.source file.name | groupby file.name'
             - name: Zeek Notice
               description: Zeek notice logs
               query: 'event.dataset:zeek.notice | groupby notice.note | groupby -sankey notice.note source.ip | groupby source.ip | groupby -sankey source.ip destination.ip | groupby destination.ip | groupby destination.port | groupby notice.message | groupby notice.sub_message | groupby source_geo.organization_name | groupby destination_geo.organization_name'
-            - name: Connections and Metadata with community_id
-              description: Network connections that include community_id
-              query: '_exists_:network.community_id | groupby event.module* | groupby -sankey event.module* event.dataset | groupby event.dataset | groupby source.ip source.port destination.ip destination.port | groupby network.protocol | groupby source_geo.organization_name source.geo.country_name | groupby destination_geo.organization_name destination.geo.country_name | groupby rule.name rule.category event.severity_label | groupby dns.query.name | groupby http.virtual_host http.uri | groupby notice.note notice.message notice.sub_message | groupby source.ip host.hostname user.name event.action event.type process.executable process.pid'
+            - name: Connections and Metadata with Community ID
+              description: Network connections that include network.community_id
+              query: '_exists_:network.community_id | groupby event.module* | groupby -sankey event.module* event.dataset | groupby event.dataset | groupby source.ip | groupby destination.ip | groupby destination.port | groupby network.protocol | groupby source_geo.organization_name | groupby source.geo.country_name | groupby destination_geo.organization_name | groupby destination.geo.country_name | groupby rule.name rule.category event.severity_label | groupby dns.query.name | groupby http.virtual_host http.uri | groupby notice.note notice.message notice.sub_message | groupby source.ip host.hostname user.name event.action event.type process.executable process.pid'
             - name: Connections seen by Zeek or Suricata
               description: Network connections logged by Zeek or Suricata
               query: 'tags:conn | groupby source.ip | groupby destination.ip | groupby destination.port | groupby -sankey destination.port network.protocol | groupby network.protocol | groupby network.transport | groupby connection.history | groupby connection.state | groupby connection.state_description | groupby source.geo.country_name | groupby destination.geo.country_name | groupby client.ip_bytes | groupby server.ip_bytes | groupby client.oui'

From 6046848ee70d81aead52d89077decb2f5aecd826 Mon Sep 17 00:00:00 2001
From: Jason Ertel <jason.ertel@securityonionsolutions.com>
Date: Thu, 4 Apr 2024 10:25:32 -0400
Subject: [PATCH 57/62] skip telemetry summary in airgap mode

---
 setup/so-whiptail | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/setup/so-whiptail b/setup/so-whiptail
index de1cfec83..90bbaf397 100755
--- a/setup/so-whiptail
+++ b/setup/so-whiptail
@@ -451,10 +451,12 @@ whiptail_end_settings() {
 		done
 	fi
 
-	if [[ $telemetry -eq 0 ]]; then
-		__append_end_msg "SOC Telemetry: enabled"
-	else
-		__append_end_msg "SOC Telemetry: disabled"
+	if [[ ! $is_airgap ]]; then
+		if [[ $telemetry -eq 0 ]]; then
+			__append_end_msg "SOC Telemetry: enabled"
+		else
+			__append_end_msg "SOC Telemetry: disabled"
+		fi
 	fi
 
 	# ADVANCED

From 49d5fa95a2953638eb75da5dc39e8ad460012041 Mon Sep 17 00:00:00 2001
From: DefensiveDepth <Josh@defensivedepth.com>
Date: Thu, 4 Apr 2024 11:26:44 -0400
Subject: [PATCH 58/62] Detections tweaks

---
 salt/soc/defaults.yaml    | 4 ++--
 salt/soc/merged.map.jinja | 5 +++++
 salt/soc/soc_soc.yaml     | 5 +++++
 3 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml
index 9bb302057..603fc5910 100644
--- a/salt/soc/defaults.yaml
+++ b/salt/soc/defaults.yaml
@@ -2049,9 +2049,9 @@ soc:
               query: "so_detection.isEnabled:false"
             - name: "Detection Type - Suricata (NIDS)"
               query: "so_detection.language:suricata"
-            - name: "Detection Type - Sigma - All"
+            - name: "Detection Type - Sigma (Elastalert) - All"
               query: "so_detection.language:sigma"
-            - name: "Detection Type - Sigma - Windows"
+            - name: "Detection Type - Sigma (Elastalert) - Windows"
               query: 'so_detection.language:sigma AND so_detection.content: "*product: windows*"'
             - name: "Detection Type - YARA (Strelka)"
               query: "so_detection.language:yara"
diff --git a/salt/soc/merged.map.jinja b/salt/soc/merged.map.jinja
index b2362a20e..4cd08a218 100644
--- a/salt/soc/merged.map.jinja
+++ b/salt/soc/merged.map.jinja
@@ -30,6 +30,11 @@
 {# since cases is not a valid soc config item and only used for the map files, remove it from being placed in the config #}
 {% do SOCMERGED.config.server.modules.pop('cases') %}
 
+{# do not automatically enable Sigma rules if install is Eval or Import #}
+{% if grains['role'] in ['so-eval', 'so-import'] %}
+  {%   do SOCMERGED.config.server.modules.elastalertengine.update({'autoEnabledSigmaRules': ""}) %}
+{% endif %}
+
 {# remove these modules if detections is disabled #}
 {% if not SOCMERGED.config.server.client.detectionsEnabled %}
 {%   do SOCMERGED.config.server.modules.pop('elastalertengine') %}
diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml
index eae52e31b..ec3f68f3f 100644
--- a/salt/soc/soc_soc.yaml
+++ b/salt/soc/soc_soc.yaml
@@ -83,6 +83,11 @@ soc:
             global: True
             advanced: True
             helpLink: sigma.html
+          autoEnabledSigmaRules:
+            description: 'Sigma rules to automatically enable on initial import. Format is $Ruleset+$Level - for example, for the core community ruleset and critical level rules: core+critical'
+            global: True
+            advanced: True
+            helpLink: sigma.html
           denyRegex:
             description: 'Regex used to filter imported Sigma rules. Deny regex takes precedence over the Allow regex setting.'
             global: True

From ca807bd6bd415b4def623a2e58ad710ac38f424b Mon Sep 17 00:00:00 2001
From: DefensiveDepth <Josh@defensivedepth.com>
Date: Thu, 4 Apr 2024 16:58:39 -0400
Subject: [PATCH 59/62] Use list not string

---
 salt/soc/merged.map.jinja | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/soc/merged.map.jinja b/salt/soc/merged.map.jinja
index 4cd08a218..ae68dc01f 100644
--- a/salt/soc/merged.map.jinja
+++ b/salt/soc/merged.map.jinja
@@ -32,7 +32,7 @@
 
 {# do not automatically enable Sigma rules if install is Eval or Import #}
 {% if grains['role'] in ['so-eval', 'so-import'] %}
-  {%   do SOCMERGED.config.server.modules.elastalertengine.update({'autoEnabledSigmaRules': ""}) %}
+  {%   do SOCMERGED.config.server.modules.elastalertengine.update({'autoEnabledSigmaRules': []}) %}
 {% endif %}
 
 {# remove these modules if detections is disabled #}

From 21f86be8ee4cbc5d1329f3bed816eba5b794248b Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Fri, 5 Apr 2024 08:03:42 -0400
Subject: [PATCH 60/62] Update so-log-check

---
 salt/common/tools/sbin/so-log-check | 1 +
 1 file changed, 1 insertion(+)

diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check
index cc5fef85b..d54c60168 100755
--- a/salt/common/tools/sbin/so-log-check
+++ b/salt/common/tools/sbin/so-log-check
@@ -235,6 +235,7 @@ exclude_log "curator.log"  # ignore since Curator has been removed
 exclude_log "playbook.log" # Playbook is removed as of 2.4.70, logs may still be on disk
 exclude_log "mysqld.log"   # MySQL is removed as of 2.4.70, logs may still be on disk
 exclude_log "soctopus.log" # Soctopus is removed as of 2.4.70, logs may still be on disk
+exclude_log "agentstatus.log" # ignore this log since it tracks agents in error state
 
 for log_file in $(cat /tmp/log_check_files); do
     status "Checking log file $log_file"

From 00cea6fb80f06248746cb709fd26b16590ee22ad Mon Sep 17 00:00:00 2001
From: Corey Ogburn <corey.ogburn@securityonionsolutions.com>
Date: Fri, 5 Apr 2024 11:22:47 -0600
Subject: [PATCH 61/62] Detection Author as a Keyword instead of Text

With Quick Actions added to Detections, as many fields should be usable as possible.
---
 .../templates/component/so/detection-mappings.json             | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/salt/elasticsearch/templates/component/so/detection-mappings.json b/salt/elasticsearch/templates/component/so/detection-mappings.json
index 9b68421e7..105a0ead2 100644
--- a/salt/elasticsearch/templates/component/so/detection-mappings.json
+++ b/salt/elasticsearch/templates/component/so/detection-mappings.json
@@ -30,7 +30,8 @@
               "type": "keyword"
             },
             "author": {
-              "type": "text"
+              "ignore_above": 1024,
+              "type": "keyword"
             },
             "description": {
               "type": "text"

From 376efab40ca344fc0fb201d30c3c067b48d22363 Mon Sep 17 00:00:00 2001
From: DefensiveDepth <Josh@defensivedepth.com>
Date: Mon, 8 Apr 2024 14:01:38 -0400
Subject: [PATCH 62/62] Ship Defender logs

---
 .../endpoints-initial/windows-defender.json   | 29 +++++++++++++++++++
 .../grid-nodes_general/system-grid-nodes.json |  6 ++++
 .../grid-nodes_heavy/system-grid-nodes.json   |  6 ++++
 .../files/ingest/.fleet_final_pipeline-1      |  1 +
 salt/soc/files/soc/sigma_so_pipeline.yaml     |  9 ++++++
 5 files changed, 51 insertions(+)
 create mode 100644 salt/elasticfleet/files/integrations/endpoints-initial/windows-defender.json

diff --git a/salt/elasticfleet/files/integrations/endpoints-initial/windows-defender.json b/salt/elasticfleet/files/integrations/endpoints-initial/windows-defender.json
new file mode 100644
index 000000000..ac4394e62
--- /dev/null
+++ b/salt/elasticfleet/files/integrations/endpoints-initial/windows-defender.json
@@ -0,0 +1,29 @@
+{
+  "package": {
+    "name": "winlog",
+    "version": ""
+  },
+  "name": "windows-defender",
+  "namespace": "default",
+  "description": "Windows Defender - Operational logs",
+  "policy_id": "endpoints-initial",
+  "inputs": {
+    "winlogs-winlog": {
+      "enabled": true,
+      "streams": {
+        "winlog.winlog": {
+          "enabled": true,
+          "vars": {
+            "channel": "Microsoft-Windows-Windows Defender/Operational",
+            "data_stream.dataset": "winlog.winlog",
+            "preserve_original_event": false,
+            "providers": [],
+            "ignore_older": "72h",
+            "language": 0,
+            "tags": []          }
+        }
+      }
+    }
+  },
+  "force": true
+}
diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/system-grid-nodes.json b/salt/elasticfleet/files/integrations/grid-nodes_general/system-grid-nodes.json
index 8e6bf7958..98204e894 100644
--- a/salt/elasticfleet/files/integrations/grid-nodes_general/system-grid-nodes.json
+++ b/salt/elasticfleet/files/integrations/grid-nodes_general/system-grid-nodes.json
@@ -16,6 +16,9 @@
             "paths": [
               "/var/log/auth.log*",
               "/var/log/secure*"
+            ],
+            "tags": [
+              "so-grid-node"
             ]
           }
         },
@@ -25,6 +28,9 @@
             "paths": [
               "/var/log/messages*",
               "/var/log/syslog*"
+            ],
+            "tags": [
+              "so-grid-node"
             ]
           }
         }
diff --git a/salt/elasticfleet/files/integrations/grid-nodes_heavy/system-grid-nodes.json b/salt/elasticfleet/files/integrations/grid-nodes_heavy/system-grid-nodes.json
index 6c42086bc..42918cc97 100644
--- a/salt/elasticfleet/files/integrations/grid-nodes_heavy/system-grid-nodes.json
+++ b/salt/elasticfleet/files/integrations/grid-nodes_heavy/system-grid-nodes.json
@@ -16,6 +16,9 @@
             "paths": [
               "/var/log/auth.log*",
               "/var/log/secure*"
+            ],
+            "tags": [
+              "so-grid-node"
             ]
           }
         },
@@ -25,6 +28,9 @@
             "paths": [
               "/var/log/messages*",
               "/var/log/syslog*"
+            ],
+            "tags": [
+              "so-grid-node"
             ]
           }
         }
diff --git a/salt/elasticsearch/files/ingest/.fleet_final_pipeline-1 b/salt/elasticsearch/files/ingest/.fleet_final_pipeline-1
index 18d078244..89216077a 100644
--- a/salt/elasticsearch/files/ingest/.fleet_final_pipeline-1
+++ b/salt/elasticsearch/files/ingest/.fleet_final_pipeline-1
@@ -83,6 +83,7 @@
     { "date":       { "if": "ctx.event?.module == 'system'", "field": "event.created", "target_field": "@timestamp", "formats": ["yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'"] } },
     { "community_id":{ "if": "ctx.event?.dataset == 'endpoint.events.network'", "ignore_failure":true }  },
     { "set":        { "if": "ctx.event?.module == 'fim'", "override": true, "field": "event.module", "value": "file_integrity" } },
+    { "rename":        { "if": "ctx.winlog?.provider_name == 'Microsoft-Windows-Windows Defender'", "ignore_missing": true, "field": "winlog.event_data.Threat Name", "target_field": "winlog.event_data.threat_name" } },
     { "remove":     { "field": [ "message2", "type", "fields", "category", "module", "dataset", "event.dataset_temp", "dataset_tag_temp", "module_temp" ], "ignore_missing": true, "ignore_failure": true } } 
   ],
   "on_failure": [
diff --git a/salt/soc/files/soc/sigma_so_pipeline.yaml b/salt/soc/files/soc/sigma_so_pipeline.yaml
index d227c3f01..312d07965 100644
--- a/salt/soc/files/soc/sigma_so_pipeline.yaml
+++ b/salt/soc/files/soc/sigma_so_pipeline.yaml
@@ -79,3 +79,12 @@ transformations:
       - type: logsource
         product: windows
         category: driver_load
+    - id: linux_security_add-fields
+      type: add_condition
+      conditions:
+        event.module: 'system'
+        event.dataset: 'system.auth'
+      rule_conditions:
+      - type: logsource
+        product: linux
+        service: auth