From ccd6b3914cda395999a0fc3819b9ed0cd0ef569d Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Thu, 6 Jun 2024 10:33:55 -0400
Subject: [PATCH 01/19] add final msg queue for soup.

---
 salt/manager/tools/sbin/soup | 29 ++++++++++++++++++++++++++++-
 1 file changed, 28 insertions(+), 1 deletion(-)

diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup
index 02c01920d..258c09ed6 100755
--- a/salt/manager/tools/sbin/soup
+++ b/salt/manager/tools/sbin/soup
@@ -19,6 +19,9 @@ SOUP_LOG=/root/soup.log
 WHATWOULDYOUSAYYAHDOHERE=soup
 whiptail_title='Security Onion UPdater'
 NOTIFYCUSTOMELASTICCONFIG=false
+# used to display messages to the user at the end of soup
+declare -a FINAL_MESSAGE_QUEUE=()
+
 
 check_err() {
   local exit_code=$1
@@ -344,6 +347,22 @@ masterunlock() {
   mv -v $BACKUPTOPFILE $TOPFILE
 }
 
+phases_pillar_2_4_80() {
+  echo "Checking if pillar value: elasticsearch.index_settings.global_overrides.index_template.phases exists"
+
+  #so-yaml.py remove /opt/so/saltstack/local/pillar/elasticsearch/soc_elasticsearch.sls elasticsearch.index_settings.global_overrides.index_template.phases
+  #if so-yaml removed stuff add this message to the FINAL_MESSAGE_QUEUE
+  read -r -d '' msg << EOM
+  Found elasticsearch.index_settings.global_overrides.index_template.phases set to:
+  so-yaml removed stuff here
+  A backup of all pillars was saved to /nsm/backup/
+  Removed unused pillar value: elasticsearch.index_settings.global_overrides.index_template.phases
+  If you want to set policies, navigate to the SOC Grid Configuration UI at elasticsearch.index_settings.global_overrides.policy.phases"
+  EOM
+
+  FINAL_MESSAGE_QUEUE+=("$msg")
+}
+
 preupgrade_changes() {
     # This function is to add any new pillar items if needed.
     echo "Checking to see if changes are needed."
@@ -603,7 +622,7 @@ up_to_2.4.70() {
 }
 
 up_to_2.4.80() {
-  echo "Nothing to do for 2.4.80"
+  phases_pillar_2_4_80
   INSTALLEDVERSION=2.4.80
 }
 
@@ -1267,6 +1286,14 @@ EOF
 
   fi
 
+# check if the FINAL_MESSAGE_QUEUE is not empty
+if (( ${#FINAL_MESSAGE_QUEUE[@]} != 0 )); then
+  for m in "${FINAL_MESSAGE_QUEUE[@]}"; do
+    echo "$m"
+    echo
+  done
+fi
+
   echo "### soup has been served at $(date) ###"
 }
 

From 6920b77b4a3425fd3eb8f6a5316d702d5f6cbf1f Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Thu, 6 Jun 2024 11:00:43 -0400
Subject: [PATCH 02/19] fix msg

---
 salt/manager/tools/sbin/soup | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup
index 258c09ed6..c510e832b 100755
--- a/salt/manager/tools/sbin/soup
+++ b/salt/manager/tools/sbin/soup
@@ -352,13 +352,13 @@ phases_pillar_2_4_80() {
 
   #so-yaml.py remove /opt/so/saltstack/local/pillar/elasticsearch/soc_elasticsearch.sls elasticsearch.index_settings.global_overrides.index_template.phases
   #if so-yaml removed stuff add this message to the FINAL_MESSAGE_QUEUE
-  read -r -d '' msg << EOM
+  read -r -d '' msg << EOF
   Found elasticsearch.index_settings.global_overrides.index_template.phases set to:
   so-yaml removed stuff here
   A backup of all pillars was saved to /nsm/backup/
   Removed unused pillar value: elasticsearch.index_settings.global_overrides.index_template.phases
-  If you want to set policies, navigate to the SOC Grid Configuration UI at elasticsearch.index_settings.global_overrides.policy.phases"
-  EOM
+  If you want to set policies, navigate to the SOC Grid Configuration UI at elasticsearch.index_settings.global_overrides.policy.phases
+EOF
 
   FINAL_MESSAGE_QUEUE+=("$msg")
 }

From 5600fed9c4854f06e98c9aed31d0ce0550d13936 Mon Sep 17 00:00:00 2001
From: Jason Ertel <jason.ertel@securityonionsolutions.com>
Date: Thu, 6 Jun 2024 11:56:07 -0400
Subject: [PATCH 03/19] add ability to retrieve yaml values via so-yaml.py;
 improve so-minion id matching

---
 pyci.sh                                 |  14 +--
 salt/manager/tools/sbin/so-minion       |   4 +-
 salt/manager/tools/sbin/so-yaml.py      |  74 +++++++++++-----
 salt/manager/tools/sbin/so-yaml_test.py | 111 +++++++++++++++++++-----
 4 files changed, 149 insertions(+), 54 deletions(-)

diff --git a/pyci.sh b/pyci.sh
index e85287063..8cbee5e75 100755
--- a/pyci.sh
+++ b/pyci.sh
@@ -15,12 +15,16 @@ TARGET_DIR=${1:-.}
 
 PATH=$PATH:/usr/local/bin
 
-if ! which pytest &> /dev/null || ! which flake8 &> /dev/null ; then
-    echo "Missing dependencies. Consider running the following command:"
-    echo "  python -m pip install flake8 pytest pytest-cov"
+if [ ! -d .venv ]; then
+    python -m venv .venv
+fi
+
+source .venv/bin/activate
+
+if ! pip install flake8 pytest pytest-cov pyyaml; then
+    echo "Unable to install dependencies."
     exit 1
 fi
 
-pip install pytest pytest-cov
 flake8 "$TARGET_DIR" "--config=${HOME_DIR}/pytest.ini"
-python3 -m pytest "--cov-config=${HOME_DIR}/pytest.ini" "--cov=$TARGET_DIR" --doctest-modules --cov-report=term --cov-fail-under=100  "$TARGET_DIR" 
\ No newline at end of file
+python3 -m pytest "--cov-config=${HOME_DIR}/pytest.ini" "--cov=$TARGET_DIR" --doctest-modules --cov-report=term --cov-fail-under=100  "$TARGET_DIR" 
diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion
index 8b563ef1d..da1a6d2a2 100755
--- a/salt/manager/tools/sbin/so-minion
+++ b/salt/manager/tools/sbin/so-minion
@@ -112,8 +112,8 @@ function testMinion() {
 	result=$?
 
 	# If this so-minion script is not running on the given minion ID, run so-test remotely on the sensor as well
-	local_id=$(lookup_grain id)
-	if [[ ! "$local_id" =~ "${MINION_ID}_" ]]; then
+	local_id=$(lookup_grain id)	
+	if [[ ! "$local_id" =~ "${MINION_ID}_" && "$local_id" != "${MINION_ID}" ]]; then
 		salt "$MINION_ID" cmd.run 'so-test'
 		result=$?
 	fi
diff --git a/salt/manager/tools/sbin/so-yaml.py b/salt/manager/tools/sbin/so-yaml.py
index 5427a2e48..cddc827b5 100755
--- a/salt/manager/tools/sbin/so-yaml.py
+++ b/salt/manager/tools/sbin/so-yaml.py
@@ -14,19 +14,20 @@ lockFile = "/tmp/so-yaml.lock"
 
 
 def showUsage(args):
-    print('Usage: {} <COMMAND> <YAML_FILE> [ARGS...]'.format(sys.argv[0]))
-    print('  General commands:')
-    print('    append         - Append a list item to a yaml key, if it exists and is a list. Requires KEY and LISTITEM args.')
-    print('    add            - Add a new key and set its value. Fails if key already exists. Requires KEY and VALUE args.')
-    print('    remove         - Removes a yaml key, if it exists. Requires KEY arg.')
-    print('    replace        - Replaces (or adds) a new key and set its value. Requires KEY and VALUE args.')
-    print('    help           - Prints this usage information.')
-    print('')
-    print('  Where:')
-    print('   YAML_FILE       - Path to the file that will be modified. Ex: /opt/so/conf/service/conf.yaml')
-    print('   KEY             - YAML key, does not support \' or " characters at this time. Ex: level1.level2')
-    print('   VALUE           - Value to set for a given key')
-    print('   LISTITEM        - Item to append to a given key\'s list value')
+    print('Usage: {} <COMMAND> <YAML_FILE> [ARGS...]'.format(sys.argv[0]), file=sys.stderr)
+    print('  General commands:', file=sys.stderr)
+    print('    append         - Append a list item to a yaml key, if it exists and is a list. Requires KEY and LISTITEM args.', file=sys.stderr)
+    print('    add            - Add a new key and set its value. Fails if key already exists. Requires KEY and VALUE args.', file=sys.stderr)
+    print('    get            - Displays (to stdout) the value stored in the given key. Requires KEY arg.', file=sys.stderr)
+    print('    remove         - Removes a yaml key, if it exists. Requires KEY arg.', file=sys.stderr)
+    print('    replace        - Replaces (or adds) a new key and set its value. Requires KEY and VALUE args.', file=sys.stderr)
+    print('    help           - Prints this usage information.', file=sys.stderr)
+    print('', file=sys.stderr)
+    print('  Where:', file=sys.stderr)
+    print('   YAML_FILE       - Path to the file that will be modified. Ex: /opt/so/conf/service/conf.yaml', file=sys.stderr)
+    print('   KEY             - YAML key, does not support \' or " characters at this time. Ex: level1.level2', file=sys.stderr)
+    print('   VALUE           - Value to set for a given key', file=sys.stderr)
+    print('   LISTITEM        - Item to append to a given key\'s list value', file=sys.stderr)
     sys.exit(1)
 
 
@@ -38,7 +39,7 @@ def loadYaml(filename):
 
 def writeYaml(filename, content):
     file = open(filename, "w")
-    return yaml.dump(content, file)
+    return yaml.safe_dump(content, file)
 
 
 def appendItem(content, key, listItem):
@@ -49,15 +50,15 @@ def appendItem(content, key, listItem):
         try:
             content[key].append(listItem)
         except AttributeError:
-            print("The existing value for the given key is not a list. No action was taken on the file.")
+            print("The existing value for the given key is not a list. No action was taken on the file.", file=sys.stderr)
             return 1
         except KeyError:
-            print("The key provided does not exist. No action was taken on the file.")
+            print("The key provided does not exist. No action was taken on the file.", file=sys.stderr)
             return 1
 
 
 def convertType(value):
-    if len(value) > 0 and (not value.startswith("0") or len(value) == 1):
+    if isinstance(value, str) and len(value) > 0 and (not value.startswith("0") or len(value) == 1):
         if "." in value:
             try:
                 value = float(value)
@@ -83,7 +84,7 @@ def append(args):
     if len(args) != 3:
         print('Missing filename, key arg, or list item to append', file=sys.stderr)
         showUsage(None)
-        return
+        return 1
 
     filename = args[0]
     key = args[1]
@@ -112,7 +113,7 @@ def add(args):
     if len(args) != 3:
         print('Missing filename, key arg, and/or value', file=sys.stderr)
         showUsage(None)
-        return
+        return 1
 
     filename = args[0]
     key = args[1]
@@ -137,7 +138,7 @@ def remove(args):
     if len(args) != 2:
         print('Missing filename or key arg', file=sys.stderr)
         showUsage(None)
-        return
+        return 1
 
     filename = args[0]
     key = args[1]
@@ -153,7 +154,7 @@ def replace(args):
     if len(args) != 3:
         print('Missing filename, key arg, and/or value', file=sys.stderr)
         showUsage(None)
-        return
+        return 1
 
     filename = args[0]
     key = args[1]
@@ -167,6 +168,32 @@ def replace(args):
     return 0
 
 
+def getKeyValue(content, key):
+    pieces = key.split(".", 1)
+    if len(pieces) > 1:
+        return getKeyValue(content[pieces[0]], pieces[1])
+    return content.get(key, None)
+
+
+def get(args):
+    if len(args) != 2:
+        print('Missing filename or key arg', file=sys.stderr)
+        showUsage(None)
+        return 1
+
+    filename = args[0]
+    key = args[1]
+
+    content = loadYaml(filename)
+    output = getKeyValue(content, key)
+    if output is None:
+        print("Not found", file=sys.stderr)
+        return 2
+
+    print(yaml.safe_dump(output))
+    return 0
+
+
 def main():
     args = sys.argv[1:]
 
@@ -178,6 +205,7 @@ def main():
         "help": showUsage,
         "add": add,
         "append": append,
+        "get": get,
         "remove": remove,
         "replace": replace,
     }
@@ -195,11 +223,11 @@ def main():
                 break
             except Exception:
                 if lockAttempts == 1:
-                    print("Waiting for lock file to be released from another process...")
+                    print("Waiting for lock file to be released from another process...", file=sys.stderr)
                 time.sleep(2)
 
         if lockAttempts == maxAttempts:
-            print("Lock file (" + lockFile + ") could not be created; proceeding without lock.")
+            print("Lock file (" + lockFile + ") could not be created; proceeding without lock.", file=sys.stderr)
 
         cmd = commands.get(args[0], showUsage)
         code = cmd(args[1:])
diff --git a/salt/manager/tools/sbin/so-yaml_test.py b/salt/manager/tools/sbin/so-yaml_test.py
index 7effabac9..ca9839e02 100644
--- a/salt/manager/tools/sbin/so-yaml_test.py
+++ b/salt/manager/tools/sbin/so-yaml_test.py
@@ -15,40 +15,40 @@ class TestRemove(unittest.TestCase):
 
     def test_main_missing_input(self):
         with patch('sys.exit', new=MagicMock()) as sysmock:
-            with patch('sys.stderr', new=StringIO()) as mock_stdout:
+            with patch('sys.stderr', new=StringIO()) as mock_stderr:
                 sys.argv = ["cmd"]
                 soyaml.main()
                 sysmock.assert_called_once_with(1)
-                self.assertIn(mock_stdout.getvalue(), "Usage:")
+                self.assertIn("Usage:", mock_stderr.getvalue())
 
     def test_main_help_locked(self):
         filename = "/tmp/so-yaml.lock"
         file = open(filename, "w")
         file.write = "fake lock file"
         with patch('sys.exit', new=MagicMock()) as sysmock:
-            with patch('sys.stderr', new=StringIO()) as mock_stdout:
+            with patch('sys.stderr', new=StringIO()) as mock_stderr:
                 with patch('time.sleep', new=MagicMock()) as mock_sleep:
                     sys.argv = ["cmd", "help"]
                     soyaml.main()
                     sysmock.assert_called()
                     mock_sleep.assert_called_with(2)
-                    self.assertIn(mock_stdout.getvalue(), "Usage:")
+                    self.assertIn("Usage:", mock_stderr.getvalue())
 
     def test_main_help(self):
         with patch('sys.exit', new=MagicMock()) as sysmock:
-            with patch('sys.stderr', new=StringIO()) as mock_stdout:
+            with patch('sys.stderr', new=StringIO()) as mock_stderr:
                 sys.argv = ["cmd", "help"]
                 soyaml.main()
                 sysmock.assert_called()
-                self.assertIn(mock_stdout.getvalue(), "Usage:")
+                self.assertIn("Usage:", mock_stderr.getvalue())
 
     def test_remove_missing_arg(self):
         with patch('sys.exit', new=MagicMock()) as sysmock:
-            with patch('sys.stderr', new=StringIO()) as mock_stdout:
+            with patch('sys.stderr', new=StringIO()) as mock_stderr:
                 sys.argv = ["cmd", "help"]
                 soyaml.remove(["file"])
                 sysmock.assert_called()
-                self.assertIn(mock_stdout.getvalue(), "Missing filename or key arg\n")
+                self.assertIn("Missing filename or key arg\n", mock_stderr.getvalue())
 
     def test_remove(self):
         filename = "/tmp/so-yaml_test-remove.yaml"
@@ -97,7 +97,7 @@ class TestRemove(unittest.TestCase):
 
     def test_remove_missing_args(self):
         with patch('sys.exit', new=MagicMock()) as sysmock:
-            with patch('sys.stderr', new=StringIO()) as mock_stdout:
+            with patch('sys.stderr', new=StringIO()) as mock_stderr:
                 filename = "/tmp/so-yaml_test-remove.yaml"
                 file = open(filename, "w")
                 file.write("{key1: { child1: 123, child2: abc }, key2: false}")
@@ -112,15 +112,15 @@ class TestRemove(unittest.TestCase):
                 expected = "{key1: { child1: 123, child2: abc }, key2: false}"
                 self.assertEqual(actual, expected)
                 sysmock.assert_called_once_with(1)
-                self.assertIn(mock_stdout.getvalue(), "Missing filename or key arg\n")
+                self.assertIn("Missing filename or key arg\n", mock_stderr.getvalue())
 
     def test_append_missing_arg(self):
         with patch('sys.exit', new=MagicMock()) as sysmock:
-            with patch('sys.stderr', new=StringIO()) as mock_stdout:
+            with patch('sys.stderr', new=StringIO()) as mock_stderr:
                 sys.argv = ["cmd", "help"]
                 soyaml.append(["file", "key"])
                 sysmock.assert_called()
-                self.assertIn(mock_stdout.getvalue(), "Missing filename, key arg, or list item to append\n")
+                self.assertIn("Missing filename, key arg, or list item to append\n", mock_stderr.getvalue())
 
     def test_append(self):
         filename = "/tmp/so-yaml_test-remove.yaml"
@@ -173,11 +173,11 @@ class TestRemove(unittest.TestCase):
         file.close()
 
         with patch('sys.exit', new=MagicMock()) as sysmock:
-            with patch('sys.stdout', new=StringIO()) as mock_stdout:
+            with patch('sys.stderr', new=StringIO()) as mock_stderr:
                 sys.argv = ["cmd", "append", filename, "key4", "h"]
                 soyaml.main()
                 sysmock.assert_called()
-                self.assertEqual(mock_stdout.getvalue(), "The key provided does not exist. No action was taken on the file.\n")
+                self.assertEqual("The key provided does not exist. No action was taken on the file.\n", mock_stderr.getvalue())
 
     def test_append_key_noexist_deep(self):
         filename = "/tmp/so-yaml_test-append.yaml"
@@ -186,11 +186,11 @@ class TestRemove(unittest.TestCase):
         file.close()
 
         with patch('sys.exit', new=MagicMock()) as sysmock:
-            with patch('sys.stdout', new=StringIO()) as mock_stdout:
+            with patch('sys.stderr', new=StringIO()) as mock_stderr:
                 sys.argv = ["cmd", "append", filename, "key1.child2.deep3", "h"]
                 soyaml.main()
                 sysmock.assert_called()
-                self.assertEqual(mock_stdout.getvalue(), "The key provided does not exist. No action was taken on the file.\n")
+                self.assertEqual("The key provided does not exist. No action was taken on the file.\n", mock_stderr.getvalue())
 
     def test_append_key_nonlist(self):
         filename = "/tmp/so-yaml_test-append.yaml"
@@ -199,11 +199,11 @@ class TestRemove(unittest.TestCase):
         file.close()
 
         with patch('sys.exit', new=MagicMock()) as sysmock:
-            with patch('sys.stdout', new=StringIO()) as mock_stdout:
+            with patch('sys.stderr', new=StringIO()) as mock_stderr:
                 sys.argv = ["cmd", "append", filename, "key1", "h"]
                 soyaml.main()
                 sysmock.assert_called()
-                self.assertEqual(mock_stdout.getvalue(), "The existing value for the given key is not a list. No action was taken on the file.\n")
+                self.assertEqual("The existing value for the given key is not a list. No action was taken on the file.\n", mock_stderr.getvalue())
 
     def test_append_key_nonlist_deep(self):
         filename = "/tmp/so-yaml_test-append.yaml"
@@ -212,11 +212,11 @@ class TestRemove(unittest.TestCase):
         file.close()
 
         with patch('sys.exit', new=MagicMock()) as sysmock:
-            with patch('sys.stdout', new=StringIO()) as mock_stdout:
+            with patch('sys.stderr', new=StringIO()) as mock_stderr:
                 sys.argv = ["cmd", "append", filename, "key1.child2.deep1", "h"]
                 soyaml.main()
                 sysmock.assert_called()
-                self.assertEqual(mock_stdout.getvalue(), "The existing value for the given key is not a list. No action was taken on the file.\n")
+                self.assertEqual("The existing value for the given key is not a list. No action was taken on the file.\n", mock_stderr.getvalue())
 
     def test_add_key(self):
         content = {}
@@ -244,11 +244,11 @@ class TestRemove(unittest.TestCase):
 
     def test_add_missing_arg(self):
         with patch('sys.exit', new=MagicMock()) as sysmock:
-            with patch('sys.stderr', new=StringIO()) as mock_stdout:
+            with patch('sys.stderr', new=StringIO()) as mock_stderr:
                 sys.argv = ["cmd", "help"]
                 soyaml.add(["file", "key"])
                 sysmock.assert_called()
-                self.assertIn(mock_stdout.getvalue(), "Missing filename, key arg, and/or value\n")
+                self.assertIn("Missing filename, key arg, and/or value\n", mock_stderr.getvalue())
 
     def test_add(self):
         filename = "/tmp/so-yaml_test-add.yaml"
@@ -296,11 +296,11 @@ class TestRemove(unittest.TestCase):
 
     def test_replace_missing_arg(self):
         with patch('sys.exit', new=MagicMock()) as sysmock:
-            with patch('sys.stderr', new=StringIO()) as mock_stdout:
+            with patch('sys.stderr', new=StringIO()) as mock_stderr:
                 sys.argv = ["cmd", "help"]
                 soyaml.replace(["file", "key"])
                 sysmock.assert_called()
-                self.assertIn(mock_stdout.getvalue(), "Missing filename, key arg, and/or value\n")
+                self.assertIn("Missing filename, key arg, and/or value\n", mock_stderr.getvalue())
 
     def test_replace(self):
         filename = "/tmp/so-yaml_test-add.yaml"
@@ -360,3 +360,66 @@ class TestRemove(unittest.TestCase):
         self.assertEqual(soyaml.convertType("false"), False)
         self.assertEqual(soyaml.convertType("FALSE"), False)
         self.assertEqual(soyaml.convertType(""), "")
+
+    def test_get_int(self):
+        with patch('sys.stdout', new=StringIO()) as mock_stdout:
+            filename = "/tmp/so-yaml_test-get.yaml"
+            file = open(filename, "w")
+            file.write("{key1: { child1: 123, child2: { deep1: 45 } }, key2: false, key3: [e,f,g]}")
+            file.close()
+
+            result = soyaml.get([filename, "key1.child2.deep1"])
+            self.assertEqual(result, 0)
+            self.assertIn("45\n...", mock_stdout.getvalue())
+
+    def test_get_str(self):
+        with patch('sys.stdout', new=StringIO()) as mock_stdout:
+            filename = "/tmp/so-yaml_test-get.yaml"
+            file = open(filename, "w")
+            file.write("{key1: { child1: 123, child2: { deep1: \"hello\" } }, key2: false, key3: [e,f,g]}")
+            file.close()
+
+            result = soyaml.get([filename, "key1.child2.deep1"])
+            self.assertEqual(result, 0)
+            self.assertIn("hello\n...", mock_stdout.getvalue())
+
+    def test_get_list(self):
+        with patch('sys.stdout', new=StringIO()) as mock_stdout:
+            filename = "/tmp/so-yaml_test-get.yaml"
+            file = open(filename, "w")
+            file.write("{key1: { child1: 123, child2: { deep1: \"hello\" } }, key2: false, key3: [e,f,g]}")
+            file.close()
+
+            result = soyaml.get([filename, "key3"])
+            self.assertEqual(result, 0)
+            self.assertIn("- e\n- f\n- g\n", mock_stdout.getvalue())
+
+    def test_get_dict(self):
+        with patch('sys.stdout', new=StringIO()) as mock_stdout:
+            filename = "/tmp/so-yaml_test-get.yaml"
+            file = open(filename, "w")
+            file.write("{key1: { child1: 123, child2: { deep1: \"hello\" } }, key2: false, key3: [e,f,g]}")
+            file.close()
+
+            result = soyaml.get([filename, "key1"])
+            self.assertEqual(result, 0)
+            self.assertIn("child1: 123\nchild2:\n  deep1: hello\n", mock_stdout.getvalue())
+
+    def test_get_missing(self):
+        with patch('sys.stdout', new=StringIO()) as mock_stdout:
+            filename = "/tmp/so-yaml_test-get.yaml"
+            file = open(filename, "w")
+            file.write("{key1: { child1: 123, child2: { deep1: 45 } }, key2: false, key3: [e,f,g]}")
+            file.close()
+
+            result = soyaml.get([filename, "key1.child2.deep3"])
+            self.assertEqual(result, 2)
+            self.assertEqual("", mock_stdout.getvalue())
+
+    def test_get_usage(self):
+        with patch('sys.exit', new=MagicMock()) as sysmock:
+            with patch('sys.stderr', new=StringIO()) as mock_stderr:
+                result = soyaml.get([])
+                self.assertEqual(result, 1)
+                self.assertIn("Missing filename or key arg", mock_stderr.getvalue())
+                sysmock.assert_called_once_with(1)

From a39c88c7b4c678aad35edd9c2c3e1fcff76a1dc4 Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Thu, 6 Jun 2024 12:56:24 -0400
Subject: [PATCH 04/19] add set to troubleshoot failure

---
 salt/manager/tools/sbin/soup | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup
index c510e832b..c09db0626 100755
--- a/salt/manager/tools/sbin/soup
+++ b/salt/manager/tools/sbin/soup
@@ -350,6 +350,7 @@ masterunlock() {
 phases_pillar_2_4_80() {
   echo "Checking if pillar value: elasticsearch.index_settings.global_overrides.index_template.phases exists"
 
+  set +e
   #so-yaml.py remove /opt/so/saltstack/local/pillar/elasticsearch/soc_elasticsearch.sls elasticsearch.index_settings.global_overrides.index_template.phases
   #if so-yaml removed stuff add this message to the FINAL_MESSAGE_QUEUE
   read -r -d '' msg << EOF
@@ -361,6 +362,7 @@ phases_pillar_2_4_80() {
 EOF
 
   FINAL_MESSAGE_QUEUE+=("$msg")
+  set -e
 }
 
 preupgrade_changes() {

From e85c3e5b27b026238d7ebf9bb39f23f2aa6fc97a Mon Sep 17 00:00:00 2001
From: Corey Ogburn <corey.ogburn@securityonionsolutions.com>
Date: Wed, 5 Jun 2024 14:45:06 -0600
Subject: [PATCH 05/19] SOC Proxy Setting

The so_proxy value we build during install is now copied to SOC's config.
---
 salt/soc/defaults.yaml    | 1 +
 salt/soc/merged.map.jinja | 5 ++++-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml
index f5628f3c3..65fb450d9 100644
--- a/salt/soc/defaults.yaml
+++ b/salt/soc/defaults.yaml
@@ -1286,6 +1286,7 @@ soc:
       maxPacketCount: 5000
       htmlDir: html
       importUploadDir: /nsm/soc/uploads
+      proxy: ''
       modules:
         cases: soc
         filedatastore:
diff --git a/salt/soc/merged.map.jinja b/salt/soc/merged.map.jinja
index f2c88fde9..4ee0eea1e 100644
--- a/salt/soc/merged.map.jinja
+++ b/salt/soc/merged.map.jinja
@@ -1,5 +1,5 @@
 {# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-   or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+   or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
    https://securityonion.net/license; you may not use this file except in compliance with the
    Elastic License 2.0. #}
 
@@ -11,6 +11,9 @@
 
 {% set SOCMERGED = salt['pillar.get']('soc', SOCDEFAULTS, merge=true) %}
 
+{% set MANAGER_PROXY = salt['pillar.get']('manager:proxy', '') %}
+{% do SOCMERGED.config.server.update({'proxy': MANAGER_PROXY}) %}
+
 {# if SOCMERGED.config.server.modules.cases == httpcase details come from the soc pillar #}
 {% if SOCMERGED.config.server.modules.cases != 'soc' %}
 {%   do SOCMERGED.config.server.modules.elastic.update({'casesEnabled': false}) %}

From 42818a9950bb7832e08099aa243fcafd71c8f75f Mon Sep 17 00:00:00 2001
From: Corey Ogburn <corey.ogburn@securityonionsolutions.com>
Date: Thu, 6 Jun 2024 13:28:07 -0600
Subject: [PATCH 06/19] Remove proxy from SOC defaults

---
 salt/soc/defaults.yaml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml
index 65fb450d9..f5628f3c3 100644
--- a/salt/soc/defaults.yaml
+++ b/salt/soc/defaults.yaml
@@ -1286,7 +1286,6 @@ soc:
       maxPacketCount: 5000
       htmlDir: html
       importUploadDir: /nsm/soc/uploads
-      proxy: ''
       modules:
         cases: soc
         filedatastore:

From f37f5ba97b4fa1b79de7d1325f639db3d7f94fc5 Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Thu, 6 Jun 2024 15:57:58 -0400
Subject: [PATCH 07/19] Update soc_suricata.yaml

---
 salt/suricata/soc_suricata.yaml | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/salt/suricata/soc_suricata.yaml b/salt/suricata/soc_suricata.yaml
index e157ff852..1ecabacd8 100644
--- a/salt/suricata/soc_suricata.yaml
+++ b/salt/suricata/soc_suricata.yaml
@@ -150,13 +150,16 @@ suricata:
             helpLink: suricata.html
     vars:
       address-groups:
-        HOME_NET: &suriaddressgroup
+        HOME_NET: 
           description: Assign a list of hosts, or networks, using CIDR notation, to this Suricata variable. The variable can then be re-used within Suricata rules. This allows for a single adjustment to the variable that will then affect all rules referencing the variable.
           regex: ^(((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\/([0-9]|[1-2][0-9]|3[0-2]))?$|^((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?))|:))|(([0-9A-Fa-f]{1,4}:){5}((:[0-9A-Fa-f]{1,4}){1,2}|:((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)|:))|(([0-9A-Fa-f]{1,4}:){4}((:[0-9A-Fa-f]{1,4}){1,3}|:((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)|:))|(([0-9A-Fa-f]{1,4}:){3}((:[0-9A-Fa-f]{1,4}){1,4}|:((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)|:))|(([0-9A-Fa-f]{1,4}:){2}((:[0-9A-Fa-f]{1,4}){1,5}|:((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)|:))|(([0-9A-Fa-f]{1,4}:){1}((:[0-9A-Fa-f]{1,4}){1,6}|:((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)|:))|(:((:[0-9A-Fa-f]{1,4}){1,7}|:)))(\/([0-9]|[1-9][0-9]|1[0-1][0-9]|12[0-8]))?$
           regexFailureMessage: You must enter a valid IP address or CIDR.
           helpLink: suricata.html
           duplicates: True
-        EXTERNAL_NET: *suriaddressgroup
+        EXTERNAL_NET: &suriaddressgroup
+          description: Assign a list of hosts, or networks, or other customization, to this Suricata variable. The variable can then be re-used within Suricata rules. This allows for a single adjustment to the variable that will then affect all rules referencing the variable.
+          helpLink: suricata.html
+          duplicates: True
         HTTP_SERVERS: *suriaddressgroup
         SMTP_SERVERS: *suriaddressgroup
         SQL_SERVERS: *suriaddressgroup

From d3b81babec949fc9631de3be24768ec1971389ea Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Thu, 6 Jun 2024 16:15:21 -0400
Subject: [PATCH 08/19] check for phases with so-yaml, remove if exists

---
 salt/manager/tools/sbin/soup | 30 +++++++++++++++++-------------
 1 file changed, 17 insertions(+), 13 deletions(-)

diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup
index c09db0626..1850c2b9b 100755
--- a/salt/manager/tools/sbin/soup
+++ b/salt/manager/tools/sbin/soup
@@ -349,20 +349,24 @@ masterunlock() {
 
 phases_pillar_2_4_80() {
   echo "Checking if pillar value: elasticsearch.index_settings.global_overrides.index_template.phases exists"
-
-  set +e
-  #so-yaml.py remove /opt/so/saltstack/local/pillar/elasticsearch/soc_elasticsearch.sls elasticsearch.index_settings.global_overrides.index_template.phases
-  #if so-yaml removed stuff add this message to the FINAL_MESSAGE_QUEUE
-  read -r -d '' msg << EOF
-  Found elasticsearch.index_settings.global_overrides.index_template.phases set to:
-  so-yaml removed stuff here
-  A backup of all pillars was saved to /nsm/backup/
-  Removed unused pillar value: elasticsearch.index_settings.global_overrides.index_template.phases
-  If you want to set policies, navigate to the SOC Grid Configuration UI at elasticsearch.index_settings.global_overrides.policy.phases
+  PHASES=$(so-yaml.py get /opt/so/saltstack/local/pillar/elasticsearch/soc_elasticsearch.sls elasticsearch.index_settings.global_overrides.index_template.phases)
+  case $? in
+    0)
+    so-yaml.py remove /opt/so/saltstack/local/pillar/elasticsearch/soc_elasticsearch.sls elasticsearch.index_settings.global_overrides.index_template.phases
+    set +e
+    read -r -d '' msg << EOF
+    Found elasticsearch.index_settings.global_overrides.index_template.phases was set to:
+    ${PHASES}
+    Removed unused pillar value: elasticsearch.index_settings.global_overrides.index_template.phases
+    If you want to set policies, navigate to the SOC Grid Configuration UI at elasticsearch.index_settings.global_overrides.policy.phases
+    A backup of all pillars was saved to /nsm/backup/
 EOF
-
-  FINAL_MESSAGE_QUEUE+=("$msg")
-  set -e
+    FINAL_MESSAGE_QUEUE+=("$msg")
+    set -e
+    ;;
+    2) echo "Pillar elasticsearch.index_settings.global_overrides.index_template.phases does not exist. No action taken." ;;
+    *) echo "so-yaml.py returned something other than 0 or 2 exit code" ;; # we shouldn't see this
+  esac
 }
 
 preupgrade_changes() {

From d39c8fae54abfb4625a0de35de2d4ee3b0d7ac83 Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Fri, 7 Jun 2024 09:01:16 -0400
Subject: [PATCH 09/19] format output

---
 salt/manager/tools/sbin/soup | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup
index 1850c2b9b..81a7545d7 100755
--- a/salt/manager/tools/sbin/soup
+++ b/salt/manager/tools/sbin/soup
@@ -354,12 +354,13 @@ phases_pillar_2_4_80() {
     0)
     so-yaml.py remove /opt/so/saltstack/local/pillar/elasticsearch/soc_elasticsearch.sls elasticsearch.index_settings.global_overrides.index_template.phases
     set +e
-    read -r -d '' msg << EOF
-    Found elasticsearch.index_settings.global_overrides.index_template.phases was set to:
-    ${PHASES}
-    Removed unused pillar value: elasticsearch.index_settings.global_overrides.index_template.phases
-    If you want to set policies, navigate to the SOC Grid Configuration UI at elasticsearch.index_settings.global_overrides.policy.phases
-    A backup of all pillars was saved to /nsm/backup/
+    read -r -d '' msg <<- EOF
+        Found elasticsearch.index_settings.global_overrides.index_template.phases was set to:
+        ${PHASES}
+
+        Removed unused pillar value: elasticsearch.index_settings.global_overrides.index_template.phases
+        To set policies, navigate to the SOC Grid Configuration UI at elasticsearch.index_settings.global_overrides.policy.phases
+        A backup of all pillar files was saved to /nsm/backup/
 EOF
     FINAL_MESSAGE_QUEUE+=("$msg")
     set -e

From f5cc35509b48bbb944b4922a94105193c89e545e Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Fri, 7 Jun 2024 11:03:26 -0400
Subject: [PATCH 10/19] fix output alignment

---
 salt/manager/tools/sbin/soup | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup
index 81a7545d7..0ab8d9d46 100755
--- a/salt/manager/tools/sbin/soup
+++ b/salt/manager/tools/sbin/soup
@@ -355,12 +355,12 @@ phases_pillar_2_4_80() {
     so-yaml.py remove /opt/so/saltstack/local/pillar/elasticsearch/soc_elasticsearch.sls elasticsearch.index_settings.global_overrides.index_template.phases
     set +e
     read -r -d '' msg <<- EOF
-        Found elasticsearch.index_settings.global_overrides.index_template.phases was set to:
-        ${PHASES}
+Found elasticsearch.index_settings.global_overrides.index_template.phases was set to:
+${PHASES}
 
-        Removed unused pillar value: elasticsearch.index_settings.global_overrides.index_template.phases
-        To set policies, navigate to the SOC Grid Configuration UI at elasticsearch.index_settings.global_overrides.policy.phases
-        A backup of all pillar files was saved to /nsm/backup/
+Removed unused pillar value: elasticsearch.index_settings.global_overrides.index_template.phases
+To set policies, navigate to the SOC Grid Configuration UI at elasticsearch.index_settings.global_overrides.policy.phases
+A backup of all pillar files was saved to /nsm/backup/
 EOF
     FINAL_MESSAGE_QUEUE+=("$msg")
     set -e

From fa063722e102cc07da2f076f64375f061bfea2ad Mon Sep 17 00:00:00 2001
From: Corey Ogburn <corey.ogburn@securityonionsolutions.com>
Date: Thu, 6 Jun 2024 16:36:09 -0600
Subject: [PATCH 11/19] RootCA and InsecureSkipVerify

New empty settings and their annotations.
---
 salt/soc/defaults.yaml | 2 ++
 salt/soc/soc_soc.yaml  | 8 ++++++++
 2 files changed, 10 insertions(+)

diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml
index f5628f3c3..03476c3f5 100644
--- a/salt/soc/defaults.yaml
+++ b/salt/soc/defaults.yaml
@@ -1286,6 +1286,8 @@ soc:
       maxPacketCount: 5000
       htmlDir: html
       importUploadDir: /nsm/soc/uploads
+      rootCA: ''
+      insecureSkipVerify: false
       modules:
         cases: soc
         filedatastore:
diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml
index 47d051e4e..ec633f773 100644
--- a/salt/soc/soc_soc.yaml
+++ b/salt/soc/soc_soc.yaml
@@ -81,6 +81,14 @@ soc:
         description: Maximum number of packets to show in the PCAP viewer. Larger values can cause more resource utilization on both the SOC server and the browser.
         global: True
         advanced: True
+      rootCA:
+        description: Root Certificate Authority (CA) public key in PEM format that SOC will use to validate outgoing requests. This is useful when the SOC server is behind a reverse proxy that performs SSL termination.
+        multiline: True
+        advanced: True
+      insecureSkipVerify:
+        description: Disable TLS verification for outgoing requests. This will make your installation less secure to MITM attacks. Recommended only for debugging purposes.
+        advanced: True
+        forcedType: bool
       modules:
         elastalertengine:
           additionalAlerters:

From 5d3fd3d389b7ed5b751d0229153c05461966f472 Mon Sep 17 00:00:00 2001
From: Corey Ogburn <corey.ogburn@securityonionsolutions.com>
Date: Fri, 7 Jun 2024 12:47:09 -0600
Subject: [PATCH 12/19] AdditionalCA and InsecureSkipVerify

New fields have been added to manager and then duplicated over to SOC's config in the same vein as how proxy was updated earlier this week.

AdditionalCA holds the PEM formatted public keys that should be trusted when making requests. It has been implemented for both Sigma's zip downloads and Sigma and Suricata's repository clones and pulls.

InsecureSkipVerify has been added to help our users troubleshoot their configuration. Setting it to true will not verify the cert on outgoing requests. Self signed, missing, or invalid certs will not throw an error.
---
 salt/manager/defaults.yaml    |  4 +++-
 salt/manager/map.jinja        |  7 +++++++
 salt/manager/soc_manager.yaml | 18 +++++++++++++++---
 salt/soc/merged.map.jinja     |  6 ++++--
 salt/soc/soc_soc.yaml         |  8 --------
 5 files changed, 29 insertions(+), 14 deletions(-)
 create mode 100644 salt/manager/map.jinja

diff --git a/salt/manager/defaults.yaml b/salt/manager/defaults.yaml
index 8bb34690e..708900af6 100644
--- a/salt/manager/defaults.yaml
+++ b/salt/manager/defaults.yaml
@@ -2,4 +2,6 @@ manager:
   reposync:
     enabled: True
     hour: 3
-    minute: 0
\ No newline at end of file
+    minute: 0
+  additionalCA: ''
+  insecureSkipVerify: False
diff --git a/salt/manager/map.jinja b/salt/manager/map.jinja
new file mode 100644
index 000000000..1ab9c12c3
--- /dev/null
+++ b/salt/manager/map.jinja
@@ -0,0 +1,7 @@
+{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+   or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+   https://securityonion.net/license; you may not use this file except in compliance with the
+   Elastic License 2.0. #}
+
+{% import_yaml 'manager/defaults.yaml' as MANAGERDEFAULTS %}
+{% set MANAGERMERGED = salt['pillar.get']('manager', MANAGERDEFAULTS.manager, merge=True) %}
\ No newline at end of file
diff --git a/salt/manager/soc_manager.yaml b/salt/manager/soc_manager.yaml
index f6461a0c7..f3346269e 100644
--- a/salt/manager/soc_manager.yaml
+++ b/salt/manager/soc_manager.yaml
@@ -7,7 +7,7 @@ manager:
     hour:
       description: The hour of the day in which the repo sync takes place.
       global: True
-      helpLink: soup.html 
+      helpLink: soup.html
     minute:
       description: The minute within the hour to run the repo sync.
       global: True
@@ -16,11 +16,23 @@ manager:
     description: Enable elastalert 1=enabled 0=disabled.
     global: True
     helpLink: elastalert.html
-  no_proxy: 
-    description: String of hosts to ignore the proxy settings for. 
+  no_proxy:
+    description: String of hosts to ignore the proxy settings for.
     global: True
     helpLink: proxy.html
   proxy:
     description: Proxy server to use for updates.
     global: True
     helpLink: proxy.html
+  additionalCA:
+    description: Additional CA certificates to trust in PEM format.
+    global: True
+    advanced: True
+    multiline: True
+    helpLink: proxy.html
+  insecureSkipVerify:
+    description: Disable TLS verification for outgoing requests. This will make your installation less secure to MITM attacks. Recommended only for debugging purposes.
+    advanced: True
+    forcedType: bool
+    global: True
+    helpLink: proxy.html
diff --git a/salt/soc/merged.map.jinja b/salt/soc/merged.map.jinja
index 4ee0eea1e..c823175cb 100644
--- a/salt/soc/merged.map.jinja
+++ b/salt/soc/merged.map.jinja
@@ -6,13 +6,15 @@
 {% from 'vars/globals.map.jinja' import GLOBALS %}
 {% from 'soc/defaults.map.jinja' import SOCDEFAULTS with context %}
 {% from 'logstash/map.jinja' import LOGSTASH_NODES %}
+{% from 'manager/map.jinja' import MANAGERMERGED %}
 {% set DOCKER_EXTRA_HOSTS = LOGSTASH_NODES %}
 {% do DOCKER_EXTRA_HOSTS.append({GLOBALS.influxdb_host:pillar.node_data[GLOBALS.influxdb_host].ip}) %}
 
 {% set SOCMERGED = salt['pillar.get']('soc', SOCDEFAULTS, merge=true) %}
 
-{% set MANAGER_PROXY = salt['pillar.get']('manager:proxy', '') %}
-{% do SOCMERGED.config.server.update({'proxy': MANAGER_PROXY}) %}
+{% do SOCMERGED.config.server.update({'proxy': MANAGERMERGED.proxy}) %}
+{% do SOCMERGED.config.server.update({'additionalCA': MANAGERMERGED.additionalCA}) %}
+{% do SOCMERGED.config.server.update({'insecureSkipVerify': MANAGERMERGED.insecureSkipVerify}) %}
 
 {# if SOCMERGED.config.server.modules.cases == httpcase details come from the soc pillar #}
 {% if SOCMERGED.config.server.modules.cases != 'soc' %}
diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml
index ec633f773..47d051e4e 100644
--- a/salt/soc/soc_soc.yaml
+++ b/salt/soc/soc_soc.yaml
@@ -81,14 +81,6 @@ soc:
         description: Maximum number of packets to show in the PCAP viewer. Larger values can cause more resource utilization on both the SOC server and the browser.
         global: True
         advanced: True
-      rootCA:
-        description: Root Certificate Authority (CA) public key in PEM format that SOC will use to validate outgoing requests. This is useful when the SOC server is behind a reverse proxy that performs SSL termination.
-        multiline: True
-        advanced: True
-      insecureSkipVerify:
-        description: Disable TLS verification for outgoing requests. This will make your installation less secure to MITM attacks. Recommended only for debugging purposes.
-        advanced: True
-        forcedType: bool
       modules:
         elastalertengine:
           additionalAlerters:

From ee696be51d5fd01276aa8db15b26af4a3a44d40c Mon Sep 17 00:00:00 2001
From: Corey Ogburn <corey.ogburn@securityonionsolutions.com>
Date: Fri, 7 Jun 2024 13:04:54 -0600
Subject: [PATCH 13/19] Remove rootCA and insecureSkipVerify from SOC defaults

---
 salt/soc/defaults.yaml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml
index 03476c3f5..f5628f3c3 100644
--- a/salt/soc/defaults.yaml
+++ b/salt/soc/defaults.yaml
@@ -1286,8 +1286,6 @@ soc:
       maxPacketCount: 5000
       htmlDir: html
       importUploadDir: /nsm/soc/uploads
-      rootCA: ''
-      insecureSkipVerify: false
       modules:
         cases: soc
         filedatastore:

From dbc56ffee787feb6f124db5e54bbd66b7c87734f Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Fri, 7 Jun 2024 15:09:09 -0400
Subject: [PATCH 14/19] Update defaults.yaml

---
 salt/firewall/defaults.yaml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/salt/firewall/defaults.yaml b/salt/firewall/defaults.yaml
index b10505956..0f7ce911a 100644
--- a/salt/firewall/defaults.yaml
+++ b/salt/firewall/defaults.yaml
@@ -1293,6 +1293,9 @@ firewall:
             beats_endpoint_ssl:
               portgroups:
                 - beats_5644
+            elastic_agent_endpoint:
+              portgroups:
+                - elastic_agent_data
             endgame:
               portgroups:
                 - endgame

From 4057238185aad7a4c8c58b226e25f23b84ef09bb Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Fri, 7 Jun 2024 15:33:49 -0400
Subject: [PATCH 15/19] Update defaults.yaml

---
 salt/firewall/defaults.yaml | 23 +++++++++++++++++------
 1 file changed, 17 insertions(+), 6 deletions(-)

diff --git a/salt/firewall/defaults.yaml b/salt/firewall/defaults.yaml
index 0f7ce911a..5d53b9864 100644
--- a/salt/firewall/defaults.yaml
+++ b/salt/firewall/defaults.yaml
@@ -1267,26 +1267,37 @@ firewall:
       chain:
         DOCKER-USER:
           hostgroups:
+            desktop:
+              portgroups:
+                - elastic_agent_data
             fleet:
               portgroups:
-                - beats_5056
+                - elastic_agent_data
+            idh:
+              portgroups:
+                - elastic_agent_data
             sensor:
               portgroups:
-                - beats_5044
-                - beats_5644
                 - elastic_agent_data
             searchnode:
               portgroups:
                 - redis
-                - beats_5644
+                - elastic_agent_data
+            standalone:
+              portgroups:
+                - redis
+                - elastic_agent_data
+            manager:
+              portgroups:
+                - elastic_agent_data
             managersearch:
               portgroups:
                 - redis
-                - beats_5644
+                - elastic_agent_data
             self:
               portgroups:
                 - redis
-                - beats_5644
+                - elastic_agent_data
             beats_endpoint:
               portgroups:
                 - beats_5044

From 0139e1827113b3ef81a20487409145116fdd46f6 Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Fri, 7 Jun 2024 16:03:21 -0400
Subject: [PATCH 16/19] additional description

---
 salt/manager/tools/sbin/soup | 1 +
 1 file changed, 1 insertion(+)

diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup
index 0ab8d9d46..6adb39f2f 100755
--- a/salt/manager/tools/sbin/soup
+++ b/salt/manager/tools/sbin/soup
@@ -1295,6 +1295,7 @@ EOF
 
 # check if the FINAL_MESSAGE_QUEUE is not empty
 if (( ${#FINAL_MESSAGE_QUEUE[@]} != 0 )); then
+echo "The following additional information specifically applies to your grid:\n"
   for m in "${FINAL_MESSAGE_QUEUE[@]}"; do
     echo "$m"
     echo

From f2f688b9b8b1e5eb1467eb140efbba6df590d87e Mon Sep 17 00:00:00 2001
From: Jason Ertel <jertel@users.noreply.github.com>
Date: Fri, 7 Jun 2024 16:18:09 -0400
Subject: [PATCH 17/19] Update soup

---
 salt/manager/tools/sbin/soup | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup
index 6adb39f2f..0d52e5c16 100755
--- a/salt/manager/tools/sbin/soup
+++ b/salt/manager/tools/sbin/soup
@@ -1295,7 +1295,7 @@ EOF
 
 # check if the FINAL_MESSAGE_QUEUE is not empty
 if (( ${#FINAL_MESSAGE_QUEUE[@]} != 0 )); then
-echo "The following additional information specifically applies to your grid:\n"
+  echo "The following additional information applies specifically to your grid:\n"
   for m in "${FINAL_MESSAGE_QUEUE[@]}"; do
     echo "$m"
     echo

From f96b82b11203e7ca400f5dfa131556fc075799e2 Mon Sep 17 00:00:00 2001
From: Jason Ertel <jason.ertel@securityonionsolutions.com>
Date: Sat, 8 Jun 2024 07:44:46 -0400
Subject: [PATCH 18/19] gracefully handle missing parent key

---
 salt/manager/tools/sbin/so-yaml.py      |  2 +-
 salt/manager/tools/sbin/so-yaml_test.py | 11 +++++++++++
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/salt/manager/tools/sbin/so-yaml.py b/salt/manager/tools/sbin/so-yaml.py
index cddc827b5..275032ee0 100755
--- a/salt/manager/tools/sbin/so-yaml.py
+++ b/salt/manager/tools/sbin/so-yaml.py
@@ -170,7 +170,7 @@ def replace(args):
 
 def getKeyValue(content, key):
     pieces = key.split(".", 1)
-    if len(pieces) > 1:
+    if len(pieces) > 1 and pieces[0] in content:
         return getKeyValue(content[pieces[0]], pieces[1])
     return content.get(key, None)
 
diff --git a/salt/manager/tools/sbin/so-yaml_test.py b/salt/manager/tools/sbin/so-yaml_test.py
index ca9839e02..5ca46cb68 100644
--- a/salt/manager/tools/sbin/so-yaml_test.py
+++ b/salt/manager/tools/sbin/so-yaml_test.py
@@ -416,6 +416,17 @@ class TestRemove(unittest.TestCase):
             self.assertEqual(result, 2)
             self.assertEqual("", mock_stdout.getvalue())
 
+    def test_get_missing_parent(self):
+        with patch('sys.stdout', new=StringIO()) as mock_stdout:
+            filename = "/tmp/so-yaml_test-get.yaml"
+            file = open(filename, "w")
+            file.write("{key1: { child1: 123, child2: { deep1: 45 } }, key2: false, key3: [e,f,g]}")
+            file.close()
+
+            result = soyaml.get([filename, "key1.child3.deep3"])
+            self.assertEqual(result, 2)
+            self.assertEqual("", mock_stdout.getvalue())
+
     def test_get_usage(self):
         with patch('sys.exit', new=MagicMock()) as sysmock:
             with patch('sys.stderr', new=StringIO()) as mock_stderr:

From f1638faa3a669c91f93e866a34ce19424fb34b7f Mon Sep 17 00:00:00 2001
From: Jason Ertel <jason.ertel@securityonionsolutions.com>
Date: Sat, 8 Jun 2024 08:18:34 -0400
Subject: [PATCH 19/19] correct placement of error check override

---
 salt/manager/tools/sbin/soup | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup
index 0d52e5c16..9ee4058f9 100755
--- a/salt/manager/tools/sbin/soup
+++ b/salt/manager/tools/sbin/soup
@@ -349,11 +349,11 @@ masterunlock() {
 
 phases_pillar_2_4_80() {
   echo "Checking if pillar value: elasticsearch.index_settings.global_overrides.index_template.phases exists"
+  set +e
   PHASES=$(so-yaml.py get /opt/so/saltstack/local/pillar/elasticsearch/soc_elasticsearch.sls elasticsearch.index_settings.global_overrides.index_template.phases)
   case $? in
     0)
     so-yaml.py remove /opt/so/saltstack/local/pillar/elasticsearch/soc_elasticsearch.sls elasticsearch.index_settings.global_overrides.index_template.phases
-    set +e
     read -r -d '' msg <<- EOF
 Found elasticsearch.index_settings.global_overrides.index_template.phases was set to:
 ${PHASES}
@@ -363,11 +363,11 @@ To set policies, navigate to the SOC Grid Configuration UI at elasticsearch.inde
 A backup of all pillar files was saved to /nsm/backup/
 EOF
     FINAL_MESSAGE_QUEUE+=("$msg")
-    set -e
     ;;
     2) echo "Pillar elasticsearch.index_settings.global_overrides.index_template.phases does not exist. No action taken." ;;
     *) echo "so-yaml.py returned something other than 0 or 2 exit code" ;; # we shouldn't see this
   esac
+  set -e
 }
 
 preupgrade_changes() {