merge 2.4/dev

Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com>

pyci.sh

@@ -15,12 +15,16 @@ TARGET_DIR=${1:-.}
 
 PATH=$PATH:/usr/local/bin
 
-if ! which pytest &> /dev/null || ! which flake8 &> /dev/null ; then
+if [ ! -d .venv ]; then
-    echo "Missing dependencies. Consider running the following command:"
+    python -m venv .venv
-    echo "  python -m pip install flake8 pytest pytest-cov"
+fi
+
+source .venv/bin/activate
+
+if ! pip install flake8 pytest pytest-cov pyyaml; then
+    echo "Unable to install dependencies."
     exit 1
 fi
 
-pip install pytest pytest-cov
 flake8 "$TARGET_DIR" "--config=${HOME_DIR}/pytest.ini"
 python3 -m pytest "--cov-config=${HOME_DIR}/pytest.ini" "--cov=$TARGET_DIR" --doctest-modules --cov-report=term --cov-fail-under=100  "$TARGET_DIR" 

salt/firewall/defaults.yaml

@@ -93,7 +93,6 @@ firewall:
       udp: []
     kafka:
       tcp:
-        - 9092
         - 9093
       udp: []
     kibana:
@@ -419,7 +418,6 @@ firewall:
                 - elastic_agent_data
                 - elastic_agent_update
                 - sensoroni
-                - kafka
             heavynode:
               portgroups:
                 - redis
@@ -433,7 +431,6 @@ firewall:
                 - elastic_agent_data
                 - elastic_agent_update
                 - sensoroni
-                - kafka
             receiver:
               portgroups:
                 - yum
@@ -616,7 +613,6 @@ firewall:
                 - elastic_agent_data
                 - elastic_agent_update
                 - sensoroni
-                - kafka
             heavynode:
               portgroups:
                 - redis
@@ -629,7 +625,6 @@ firewall:
                 - elastic_agent_data
                 - elastic_agent_update
                 - sensoroni
-                - kafka
             receiver:
               portgroups:
                 - yum
@@ -639,6 +634,7 @@ firewall:
                 - elastic_agent_data
                 - elastic_agent_update
                 - sensoroni
+                - kafka
             analyst:
               portgroups:
                 - nginx
@@ -818,7 +814,6 @@ firewall:
                 - redis
                 - elasticsearch_rest
                 - elasticsearch_node
-                - kafka
             heavynode:
               portgroups:
                 - docker_registry
@@ -828,7 +823,6 @@ firewall:
                 - redis
                 - elasticsearch_rest
                 - elasticsearch_node
-                - kafka
             receiver:
               portgroups:
                 - yum
@@ -838,6 +832,7 @@ firewall:
                 - elastic_agent_data
                 - elastic_agent_update
                 - sensoroni
+                - kafka
             analyst:
               portgroups:
                 - nginx
@@ -1282,38 +1277,50 @@ firewall:
       chain:
         DOCKER-USER:
           hostgroups:
+            desktop:
+              portgroups:
+                - elastic_agent_data
             fleet:
               portgroups:
-                - beats_5056
+                - elastic_agent_data
+            idh:
+              portgroups:
+                - elastic_agent_data
             sensor:
               portgroups:
-                - beats_5044
-                - beats_5644
                 - elastic_agent_data
-                - kafka
             searchnode:
               portgroups:
                 - redis
-                - beats_5644
+                - elastic_agent_data
+            standalone:
+              portgroups:
+                - redis
+                - elastic_agent_data
                 - kafka
             manager:
               portgroups:
+                - elastic_agent_data
                 - kafka
             managersearch:
               portgroups:
                 - redis
-                - beats_5644
+                - elastic_agent_data
                 - kafka
             self:
               portgroups:
                 - redis
-                - beats_5644
+                - elastic_agent_data
+                - kafka
             beats_endpoint:
               portgroups:
                 - beats_5044
             beats_endpoint_ssl:
               portgroups:
                 - beats_5644
+            elastic_agent_endpoint:
+              portgroups:
+                - elastic_agent_data
             endgame:
               portgroups:
                 - endgame

salt/manager/defaults.yaml

@@ -3,3 +3,5 @@ manager:
     enabled: True
     hour: 3
     minute: 0
+  additionalCA: ''
+  insecureSkipVerify: False

salt/manager/map.jinja

@@ -0,0 +1,7 @@
+{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+   or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+   https://securityonion.net/license; you may not use this file except in compliance with the
+   Elastic License 2.0. #}
+
+{% import_yaml 'manager/defaults.yaml' as MANAGERDEFAULTS %}
+{% set MANAGERMERGED = salt['pillar.get']('manager', MANAGERDEFAULTS.manager, merge=True) %}

salt/manager/soc_manager.yaml

@@ -24,3 +24,15 @@ manager:
     description: Proxy server to use for updates.
     global: True
     helpLink: proxy.html
+  additionalCA:
+    description: Additional CA certificates to trust in PEM format.
+    global: True
+    advanced: True
+    multiline: True
+    helpLink: proxy.html
+  insecureSkipVerify:
+    description: Disable TLS verification for outgoing requests. This will make your installation less secure to MITM attacks. Recommended only for debugging purposes.
+    advanced: True
+    forcedType: bool
+    global: True
+    helpLink: proxy.html

salt/manager/tools/sbin/so-minion

@@ -113,7 +113,7 @@ function testMinion() {
 
 	# If this so-minion script is not running on the given minion ID, run so-test remotely on the sensor as well
 	local_id=$(lookup_grain id)	
-	if [[ ! "$local_id" =~ "${MINION_ID}_" ]]; then
+	if [[ ! "$local_id" =~ "${MINION_ID}_" && "$local_id" != "${MINION_ID}" ]]; then
 		salt "$MINION_ID" cmd.run 'so-test'
 		result=$?
 	fi

salt/manager/tools/sbin/so-yaml.py

@@ -14,19 +14,20 @@ lockFile = "/tmp/so-yaml.lock"
 
 
 def showUsage(args):
-    print('Usage: {} <COMMAND> <YAML_FILE> [ARGS...]'.format(sys.argv[0]))
+    print('Usage: {} <COMMAND> <YAML_FILE> [ARGS...]'.format(sys.argv[0]), file=sys.stderr)
-    print('  General commands:')
+    print('  General commands:', file=sys.stderr)
-    print('    append         - Append a list item to a yaml key, if it exists and is a list. Requires KEY and LISTITEM args.')
+    print('    append         - Append a list item to a yaml key, if it exists and is a list. Requires KEY and LISTITEM args.', file=sys.stderr)
-    print('    add            - Add a new key and set its value. Fails if key already exists. Requires KEY and VALUE args.')
+    print('    add            - Add a new key and set its value. Fails if key already exists. Requires KEY and VALUE args.', file=sys.stderr)
-    print('    remove         - Removes a yaml key, if it exists. Requires KEY arg.')
+    print('    get            - Displays (to stdout) the value stored in the given key. Requires KEY arg.', file=sys.stderr)
-    print('    replace        - Replaces (or adds) a new key and set its value. Requires KEY and VALUE args.')
+    print('    remove         - Removes a yaml key, if it exists. Requires KEY arg.', file=sys.stderr)
-    print('    help           - Prints this usage information.')
+    print('    replace        - Replaces (or adds) a new key and set its value. Requires KEY and VALUE args.', file=sys.stderr)
-    print('')
+    print('    help           - Prints this usage information.', file=sys.stderr)
-    print('  Where:')
+    print('', file=sys.stderr)
-    print('   YAML_FILE       - Path to the file that will be modified. Ex: /opt/so/conf/service/conf.yaml')
+    print('  Where:', file=sys.stderr)
-    print('   KEY             - YAML key, does not support \' or " characters at this time. Ex: level1.level2')
+    print('   YAML_FILE       - Path to the file that will be modified. Ex: /opt/so/conf/service/conf.yaml', file=sys.stderr)
-    print('   VALUE           - Value to set for a given key')
+    print('   KEY             - YAML key, does not support \' or " characters at this time. Ex: level1.level2', file=sys.stderr)
-    print('   LISTITEM        - Item to append to a given key\'s list value')
+    print('   VALUE           - Value to set for a given key', file=sys.stderr)
+    print('   LISTITEM        - Item to append to a given key\'s list value', file=sys.stderr)
     sys.exit(1)
 
 
@@ -38,7 +39,7 @@ def loadYaml(filename):
 
 def writeYaml(filename, content):
     file = open(filename, "w")
-    return yaml.dump(content, file)
+    return yaml.safe_dump(content, file)
 
 
 def appendItem(content, key, listItem):
@@ -49,15 +50,15 @@ def appendItem(content, key, listItem):
         try:
             content[key].append(listItem)
         except AttributeError:
-            print("The existing value for the given key is not a list. No action was taken on the file.")
+            print("The existing value for the given key is not a list. No action was taken on the file.", file=sys.stderr)
             return 1
         except KeyError:
-            print("The key provided does not exist. No action was taken on the file.")
+            print("The key provided does not exist. No action was taken on the file.", file=sys.stderr)
             return 1
 
 
 def convertType(value):
-    if len(value) > 0 and (not value.startswith("0") or len(value) == 1):
+    if isinstance(value, str) and len(value) > 0 and (not value.startswith("0") or len(value) == 1):
         if "." in value:
             try:
                 value = float(value)
@@ -83,7 +84,7 @@ def append(args):
     if len(args) != 3:
         print('Missing filename, key arg, or list item to append', file=sys.stderr)
         showUsage(None)
-        return
+        return 1
 
     filename = args[0]
     key = args[1]
@@ -112,7 +113,7 @@ def add(args):
     if len(args) != 3:
         print('Missing filename, key arg, and/or value', file=sys.stderr)
         showUsage(None)
-        return
+        return 1
 
     filename = args[0]
     key = args[1]
@@ -137,7 +138,7 @@ def remove(args):
     if len(args) != 2:
         print('Missing filename or key arg', file=sys.stderr)
         showUsage(None)
-        return
+        return 1
 
     filename = args[0]
     key = args[1]
@@ -153,7 +154,7 @@ def replace(args):
     if len(args) != 3:
         print('Missing filename, key arg, and/or value', file=sys.stderr)
         showUsage(None)
-        return
+        return 1
 
     filename = args[0]
     key = args[1]
@@ -167,6 +168,32 @@ def replace(args):
     return 0
 
 
+def getKeyValue(content, key):
+    pieces = key.split(".", 1)
+    if len(pieces) > 1 and pieces[0] in content:
+        return getKeyValue(content[pieces[0]], pieces[1])
+    return content.get(key, None)
+
+
+def get(args):
+    if len(args) != 2:
+        print('Missing filename or key arg', file=sys.stderr)
+        showUsage(None)
+        return 1
+
+    filename = args[0]
+    key = args[1]
+
+    content = loadYaml(filename)
+    output = getKeyValue(content, key)
+    if output is None:
+        print("Not found", file=sys.stderr)
+        return 2
+
+    print(yaml.safe_dump(output))
+    return 0
+
+
 def main():
     args = sys.argv[1:]
 
@@ -178,6 +205,7 @@ def main():
         "help": showUsage,
         "add": add,
         "append": append,
+        "get": get,
         "remove": remove,
         "replace": replace,
     }
@@ -195,11 +223,11 @@ def main():
                 break
             except Exception:
                 if lockAttempts == 1:
-                    print("Waiting for lock file to be released from another process...")
+                    print("Waiting for lock file to be released from another process...", file=sys.stderr)
                 time.sleep(2)
 
         if lockAttempts == maxAttempts:
-            print("Lock file (" + lockFile + ") could not be created; proceeding without lock.")
+            print("Lock file (" + lockFile + ") could not be created; proceeding without lock.", file=sys.stderr)
 
         cmd = commands.get(args[0], showUsage)
         code = cmd(args[1:])

salt/manager/tools/sbin/so-yaml_test.py

@@ -15,40 +15,40 @@ class TestRemove(unittest.TestCase):
 
     def test_main_missing_input(self):
         with patch('sys.exit', new=MagicMock()) as sysmock:
-            with patch('sys.stderr', new=StringIO()) as mock_stdout:
+            with patch('sys.stderr', new=StringIO()) as mock_stderr:
                 sys.argv = ["cmd"]
                 soyaml.main()
                 sysmock.assert_called_once_with(1)
-                self.assertIn(mock_stdout.getvalue(), "Usage:")
+                self.assertIn("Usage:", mock_stderr.getvalue())
 
     def test_main_help_locked(self):
         filename = "/tmp/so-yaml.lock"
         file = open(filename, "w")
         file.write = "fake lock file"
         with patch('sys.exit', new=MagicMock()) as sysmock:
-            with patch('sys.stderr', new=StringIO()) as mock_stdout:
+            with patch('sys.stderr', new=StringIO()) as mock_stderr:
                 with patch('time.sleep', new=MagicMock()) as mock_sleep:
                     sys.argv = ["cmd", "help"]
                     soyaml.main()
                     sysmock.assert_called()
                     mock_sleep.assert_called_with(2)
-                    self.assertIn(mock_stdout.getvalue(), "Usage:")
+                    self.assertIn("Usage:", mock_stderr.getvalue())
 
     def test_main_help(self):
         with patch('sys.exit', new=MagicMock()) as sysmock:
-            with patch('sys.stderr', new=StringIO()) as mock_stdout:
+            with patch('sys.stderr', new=StringIO()) as mock_stderr:
                 sys.argv = ["cmd", "help"]
                 soyaml.main()
                 sysmock.assert_called()
-                self.assertIn(mock_stdout.getvalue(), "Usage:")
+                self.assertIn("Usage:", mock_stderr.getvalue())
 
     def test_remove_missing_arg(self):
         with patch('sys.exit', new=MagicMock()) as sysmock:
-            with patch('sys.stderr', new=StringIO()) as mock_stdout:
+            with patch('sys.stderr', new=StringIO()) as mock_stderr:
                 sys.argv = ["cmd", "help"]
                 soyaml.remove(["file"])
                 sysmock.assert_called()
-                self.assertIn(mock_stdout.getvalue(), "Missing filename or key arg\n")
+                self.assertIn("Missing filename or key arg\n", mock_stderr.getvalue())
 
     def test_remove(self):
         filename = "/tmp/so-yaml_test-remove.yaml"
@@ -97,7 +97,7 @@ class TestRemove(unittest.TestCase):
 
     def test_remove_missing_args(self):
         with patch('sys.exit', new=MagicMock()) as sysmock:
-            with patch('sys.stderr', new=StringIO()) as mock_stdout:
+            with patch('sys.stderr', new=StringIO()) as mock_stderr:
                 filename = "/tmp/so-yaml_test-remove.yaml"
                 file = open(filename, "w")
                 file.write("{key1: { child1: 123, child2: abc }, key2: false}")
@@ -112,15 +112,15 @@ class TestRemove(unittest.TestCase):
                 expected = "{key1: { child1: 123, child2: abc }, key2: false}"
                 self.assertEqual(actual, expected)
                 sysmock.assert_called_once_with(1)
-                self.assertIn(mock_stdout.getvalue(), "Missing filename or key arg\n")
+                self.assertIn("Missing filename or key arg\n", mock_stderr.getvalue())
 
     def test_append_missing_arg(self):
         with patch('sys.exit', new=MagicMock()) as sysmock:
-            with patch('sys.stderr', new=StringIO()) as mock_stdout:
+            with patch('sys.stderr', new=StringIO()) as mock_stderr:
                 sys.argv = ["cmd", "help"]
                 soyaml.append(["file", "key"])
                 sysmock.assert_called()
-                self.assertIn(mock_stdout.getvalue(), "Missing filename, key arg, or list item to append\n")
+                self.assertIn("Missing filename, key arg, or list item to append\n", mock_stderr.getvalue())
 
     def test_append(self):
         filename = "/tmp/so-yaml_test-remove.yaml"
@@ -173,11 +173,11 @@ class TestRemove(unittest.TestCase):
         file.close()
 
         with patch('sys.exit', new=MagicMock()) as sysmock:
-            with patch('sys.stdout', new=StringIO()) as mock_stdout:
+            with patch('sys.stderr', new=StringIO()) as mock_stderr:
                 sys.argv = ["cmd", "append", filename, "key4", "h"]
                 soyaml.main()
                 sysmock.assert_called()
-                self.assertEqual(mock_stdout.getvalue(), "The key provided does not exist. No action was taken on the file.\n")
+                self.assertEqual("The key provided does not exist. No action was taken on the file.\n", mock_stderr.getvalue())
 
     def test_append_key_noexist_deep(self):
         filename = "/tmp/so-yaml_test-append.yaml"
@@ -186,11 +186,11 @@ class TestRemove(unittest.TestCase):
         file.close()
 
         with patch('sys.exit', new=MagicMock()) as sysmock:
-            with patch('sys.stdout', new=StringIO()) as mock_stdout:
+            with patch('sys.stderr', new=StringIO()) as mock_stderr:
                 sys.argv = ["cmd", "append", filename, "key1.child2.deep3", "h"]
                 soyaml.main()
                 sysmock.assert_called()
-                self.assertEqual(mock_stdout.getvalue(), "The key provided does not exist. No action was taken on the file.\n")
+                self.assertEqual("The key provided does not exist. No action was taken on the file.\n", mock_stderr.getvalue())
 
     def test_append_key_nonlist(self):
         filename = "/tmp/so-yaml_test-append.yaml"
@@ -199,11 +199,11 @@ class TestRemove(unittest.TestCase):
         file.close()
 
         with patch('sys.exit', new=MagicMock()) as sysmock:
-            with patch('sys.stdout', new=StringIO()) as mock_stdout:
+            with patch('sys.stderr', new=StringIO()) as mock_stderr:
                 sys.argv = ["cmd", "append", filename, "key1", "h"]
                 soyaml.main()
                 sysmock.assert_called()
-                self.assertEqual(mock_stdout.getvalue(), "The existing value for the given key is not a list. No action was taken on the file.\n")
+                self.assertEqual("The existing value for the given key is not a list. No action was taken on the file.\n", mock_stderr.getvalue())
 
     def test_append_key_nonlist_deep(self):
         filename = "/tmp/so-yaml_test-append.yaml"
@@ -212,11 +212,11 @@ class TestRemove(unittest.TestCase):
         file.close()
 
         with patch('sys.exit', new=MagicMock()) as sysmock:
-            with patch('sys.stdout', new=StringIO()) as mock_stdout:
+            with patch('sys.stderr', new=StringIO()) as mock_stderr:
                 sys.argv = ["cmd", "append", filename, "key1.child2.deep1", "h"]
                 soyaml.main()
                 sysmock.assert_called()
-                self.assertEqual(mock_stdout.getvalue(), "The existing value for the given key is not a list. No action was taken on the file.\n")
+                self.assertEqual("The existing value for the given key is not a list. No action was taken on the file.\n", mock_stderr.getvalue())
 
     def test_add_key(self):
         content = {}
@@ -244,11 +244,11 @@ class TestRemove(unittest.TestCase):
 
     def test_add_missing_arg(self):
         with patch('sys.exit', new=MagicMock()) as sysmock:
-            with patch('sys.stderr', new=StringIO()) as mock_stdout:
+            with patch('sys.stderr', new=StringIO()) as mock_stderr:
                 sys.argv = ["cmd", "help"]
                 soyaml.add(["file", "key"])
                 sysmock.assert_called()
-                self.assertIn(mock_stdout.getvalue(), "Missing filename, key arg, and/or value\n")
+                self.assertIn("Missing filename, key arg, and/or value\n", mock_stderr.getvalue())
 
     def test_add(self):
         filename = "/tmp/so-yaml_test-add.yaml"
@@ -296,11 +296,11 @@ class TestRemove(unittest.TestCase):
 
     def test_replace_missing_arg(self):
         with patch('sys.exit', new=MagicMock()) as sysmock:
-            with patch('sys.stderr', new=StringIO()) as mock_stdout:
+            with patch('sys.stderr', new=StringIO()) as mock_stderr:
                 sys.argv = ["cmd", "help"]
                 soyaml.replace(["file", "key"])
                 sysmock.assert_called()
-                self.assertIn(mock_stdout.getvalue(), "Missing filename, key arg, and/or value\n")
+                self.assertIn("Missing filename, key arg, and/or value\n", mock_stderr.getvalue())
 
     def test_replace(self):
         filename = "/tmp/so-yaml_test-add.yaml"
@@ -360,3 +360,77 @@ class TestRemove(unittest.TestCase):
         self.assertEqual(soyaml.convertType("false"), False)
         self.assertEqual(soyaml.convertType("FALSE"), False)
         self.assertEqual(soyaml.convertType(""), "")
+
+    def test_get_int(self):
+        with patch('sys.stdout', new=StringIO()) as mock_stdout:
+            filename = "/tmp/so-yaml_test-get.yaml"
+            file = open(filename, "w")
+            file.write("{key1: { child1: 123, child2: { deep1: 45 } }, key2: false, key3: [e,f,g]}")
+            file.close()
+
+            result = soyaml.get([filename, "key1.child2.deep1"])
+            self.assertEqual(result, 0)
+            self.assertIn("45\n...", mock_stdout.getvalue())
+
+    def test_get_str(self):
+        with patch('sys.stdout', new=StringIO()) as mock_stdout:
+            filename = "/tmp/so-yaml_test-get.yaml"
+            file = open(filename, "w")
+            file.write("{key1: { child1: 123, child2: { deep1: \"hello\" } }, key2: false, key3: [e,f,g]}")
+            file.close()
+
+            result = soyaml.get([filename, "key1.child2.deep1"])
+            self.assertEqual(result, 0)
+            self.assertIn("hello\n...", mock_stdout.getvalue())
+
+    def test_get_list(self):
+        with patch('sys.stdout', new=StringIO()) as mock_stdout:
+            filename = "/tmp/so-yaml_test-get.yaml"
+            file = open(filename, "w")
+            file.write("{key1: { child1: 123, child2: { deep1: \"hello\" } }, key2: false, key3: [e,f,g]}")
+            file.close()
+
+            result = soyaml.get([filename, "key3"])
+            self.assertEqual(result, 0)
+            self.assertIn("- e\n- f\n- g\n", mock_stdout.getvalue())
+
+    def test_get_dict(self):
+        with patch('sys.stdout', new=StringIO()) as mock_stdout:
+            filename = "/tmp/so-yaml_test-get.yaml"
+            file = open(filename, "w")
+            file.write("{key1: { child1: 123, child2: { deep1: \"hello\" } }, key2: false, key3: [e,f,g]}")
+            file.close()
+
+            result = soyaml.get([filename, "key1"])
+            self.assertEqual(result, 0)
+            self.assertIn("child1: 123\nchild2:\n  deep1: hello\n", mock_stdout.getvalue())
+
+    def test_get_missing(self):
+        with patch('sys.stdout', new=StringIO()) as mock_stdout:
+            filename = "/tmp/so-yaml_test-get.yaml"
+            file = open(filename, "w")
+            file.write("{key1: { child1: 123, child2: { deep1: 45 } }, key2: false, key3: [e,f,g]}")
+            file.close()
+
+            result = soyaml.get([filename, "key1.child2.deep3"])
+            self.assertEqual(result, 2)
+            self.assertEqual("", mock_stdout.getvalue())
+
+    def test_get_missing_parent(self):
+        with patch('sys.stdout', new=StringIO()) as mock_stdout:
+            filename = "/tmp/so-yaml_test-get.yaml"
+            file = open(filename, "w")
+            file.write("{key1: { child1: 123, child2: { deep1: 45 } }, key2: false, key3: [e,f,g]}")
+            file.close()
+
+            result = soyaml.get([filename, "key1.child3.deep3"])
+            self.assertEqual(result, 2)
+            self.assertEqual("", mock_stdout.getvalue())
+
+    def test_get_usage(self):
+        with patch('sys.exit', new=MagicMock()) as sysmock:
+            with patch('sys.stderr', new=StringIO()) as mock_stderr:
+                result = soyaml.get([])
+                self.assertEqual(result, 1)
+                self.assertIn("Missing filename or key arg", mock_stderr.getvalue())
+                sysmock.assert_called_once_with(1)

salt/manager/tools/sbin/soup

@@ -19,6 +19,9 @@ SOUP_LOG=/root/soup.log
 WHATWOULDYOUSAYYAHDOHERE=soup
 whiptail_title='Security Onion UPdater'
 NOTIFYCUSTOMELASTICCONFIG=false
+# used to display messages to the user at the end of soup
+declare -a FINAL_MESSAGE_QUEUE=()
+
 
 check_err() {
   local exit_code=$1
@@ -344,6 +347,29 @@ masterunlock() {
   mv -v $BACKUPTOPFILE $TOPFILE
 }
 
+phases_pillar_2_4_80() {
+  echo "Checking if pillar value: elasticsearch.index_settings.global_overrides.index_template.phases exists"
+  set +e
+  PHASES=$(so-yaml.py get /opt/so/saltstack/local/pillar/elasticsearch/soc_elasticsearch.sls elasticsearch.index_settings.global_overrides.index_template.phases)
+  case $? in
+    0)
+    so-yaml.py remove /opt/so/saltstack/local/pillar/elasticsearch/soc_elasticsearch.sls elasticsearch.index_settings.global_overrides.index_template.phases
+    read -r -d '' msg <<- EOF
+Found elasticsearch.index_settings.global_overrides.index_template.phases was set to:
+${PHASES}
+
+Removed unused pillar value: elasticsearch.index_settings.global_overrides.index_template.phases
+To set policies, navigate to the SOC Grid Configuration UI at elasticsearch.index_settings.global_overrides.policy.phases
+A backup of all pillar files was saved to /nsm/backup/
+EOF
+    FINAL_MESSAGE_QUEUE+=("$msg")
+    ;;
+    2) echo "Pillar elasticsearch.index_settings.global_overrides.index_template.phases does not exist. No action taken." ;;
+    *) echo "so-yaml.py returned something other than 0 or 2 exit code" ;; # we shouldn't see this
+  esac
+  set -e
+}
+
 preupgrade_changes() {
     # This function is to add any new pillar items if needed.
     echo "Checking to see if changes are needed."
@@ -604,6 +630,7 @@ up_to_2.4.70() {
 }
 
 up_to_2.4.80() {
+  phases_pillar_2_4_80
   # Kafka configuration changes
 
   # Global pipeline changes to REDIS or KAFKA
@@ -1282,6 +1309,15 @@ EOF
 
   fi
 
+# check if the FINAL_MESSAGE_QUEUE is not empty
+if (( ${#FINAL_MESSAGE_QUEUE[@]} != 0 )); then
+  echo "The following additional information applies specifically to your grid:\n"
+  for m in "${FINAL_MESSAGE_QUEUE[@]}"; do
+    echo "$m"
+    echo
+  done
+fi
+
   echo "### soup has been served at $(date) ###"
 }
 

salt/soc/merged.map.jinja

@@ -6,11 +6,16 @@
 {% from 'vars/globals.map.jinja' import GLOBALS %}
 {% from 'soc/defaults.map.jinja' import SOCDEFAULTS with context %}
 {% from 'logstash/map.jinja' import LOGSTASH_NODES %}
+{% from 'manager/map.jinja' import MANAGERMERGED %}
 {% set DOCKER_EXTRA_HOSTS = LOGSTASH_NODES %}
 {% do DOCKER_EXTRA_HOSTS.append({GLOBALS.influxdb_host:pillar.node_data[GLOBALS.influxdb_host].ip}) %}
 
 {% set SOCMERGED = salt['pillar.get']('soc', SOCDEFAULTS, merge=true) %}
 
+{% do SOCMERGED.config.server.update({'proxy': MANAGERMERGED.proxy}) %}
+{% do SOCMERGED.config.server.update({'additionalCA': MANAGERMERGED.additionalCA}) %}
+{% do SOCMERGED.config.server.update({'insecureSkipVerify': MANAGERMERGED.insecureSkipVerify}) %}
+
 {# if SOCMERGED.config.server.modules.cases == httpcase details come from the soc pillar #}
 {% if SOCMERGED.config.server.modules.cases != 'soc' %}
 {%   do SOCMERGED.config.server.modules.elastic.update({'casesEnabled': false}) %}

salt/suricata/soc_suricata.yaml

@@ -150,13 +150,16 @@ suricata:
             helpLink: suricata.html
     vars:
       address-groups:
-        HOME_NET: &suriaddressgroup
+        HOME_NET: 
           description: Assign a list of hosts, or networks, using CIDR notation, to this Suricata variable. The variable can then be re-used within Suricata rules. This allows for a single adjustment to the variable that will then affect all rules referencing the variable.
           regex: ^(((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\/([0-9]|[1-2][0-9]|3[0-2]))?$|^((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?))|:))|(([0-9A-Fa-f]{1,4}:){5}((:[0-9A-Fa-f]{1,4}){1,2}|:((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)|:))|(([0-9A-Fa-f]{1,4}:){4}((:[0-9A-Fa-f]{1,4}){1,3}|:((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)|:))|(([0-9A-Fa-f]{1,4}:){3}((:[0-9A-Fa-f]{1,4}){1,4}|:((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)|:))|(([0-9A-Fa-f]{1,4}:){2}((:[0-9A-Fa-f]{1,4}){1,5}|:((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)|:))|(([0-9A-Fa-f]{1,4}:){1}((:[0-9A-Fa-f]{1,4}){1,6}|:((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)|:))|(:((:[0-9A-Fa-f]{1,4}){1,7}|:)))(\/([0-9]|[1-9][0-9]|1[0-1][0-9]|12[0-8]))?$
           regexFailureMessage: You must enter a valid IP address or CIDR.
           helpLink: suricata.html
           duplicates: True
-        EXTERNAL_NET: *suriaddressgroup
+        EXTERNAL_NET: &suriaddressgroup
+          description: Assign a list of hosts, or networks, or other customization, to this Suricata variable. The variable can then be re-used within Suricata rules. This allows for a single adjustment to the variable that will then affect all rules referencing the variable.
+          helpLink: suricata.html
+          duplicates: True
         HTTP_SERVERS: *suriaddressgroup
         SMTP_SERVERS: *suriaddressgroup
         SQL_SERVERS: *suriaddressgroup