Merge pull request #12325 from Security-Onion-Solutions/salt3006.6

Salt3006.6

files/salt/master/master

@@ -41,6 +41,7 @@ file_roots:
   base:
     - /opt/so/saltstack/local/salt
     - /opt/so/saltstack/default/salt
+    - /opt/so/rules
 
 
 # The master_roots setting configures a master-only copy of the file_roots dictionary,

salt/manager/tools/sbin/so-yaml.py

@@ -16,12 +16,14 @@ lockFile = "/tmp/so-yaml.lock"
 def showUsage(args):
     print('Usage: {} <COMMAND> <YAML_FILE> [ARGS...]'.format(sys.argv[0]))
     print('  General commands:')
+    print('    append         - Append a list item to a yaml key, if it exists and is a list. Requires KEY and LISTITEM args.')
     print('    remove         - Removes a yaml key, if it exists. Requires KEY arg.')
     print('    help           - Prints this usage information.')
     print('')
     print('  Where:')
     print('   YAML_FILE       - Path to the file that will be modified. Ex: /opt/so/conf/service/conf.yaml')
     print('   KEY             - YAML key, does not support \' or " characters at this time. Ex: level1.level2')
+    print('   LISTITEM        - Item to add to the list.')
     sys.exit(1)
 
 
@@ -35,6 +37,35 @@ def writeYaml(filename, content):
     file = open(filename, "w")
     return yaml.dump(content, file)
 
+def appendItem(content, key, listItem):
+    pieces = key.split(".", 1)
+    if len(pieces) > 1:
+        appendItem(content[pieces[0]], pieces[1], listItem)
+    else:
+        try:
+            content[key].append(listItem)
+        except AttributeError:
+            print("The existing value for the given key is not a list. No action was taken on the file.")
+            return 1
+        except KeyError:
+            print("The key provided does not exist. No action was taken on the file.")
+            return 1
+
+def append(args):
+    if len(args) != 3:
+        print('Missing filename, key arg, or list item to append', file=sys.stderr)
+        showUsage(None)
+        return
+
+    filename = args[0]
+    key = args[1]
+    listItem = args[2]
+
+    content = loadYaml(filename)
+    appendItem(content, key, listItem)
+    writeYaml(filename, content)
+
+    return 0
 
 def removeKey(content, key):
     pieces = key.split(".", 1)
@@ -69,6 +100,7 @@ def main():
 
     commands = {
         "help": showUsage,
+        "append": append,
         "remove": remove,
     }
 

salt/manager/tools/sbin/so-yaml_test.py

@@ -105,3 +105,99 @@ class TestRemove(unittest.TestCase):
                 self.assertEqual(actual, expected)
                 sysmock.assert_called_once_with(1)
                 self.assertIn(mock_stdout.getvalue(), "Missing filename or key arg\n")
+
+    def test_append(self):
+        filename = "/tmp/so-yaml_test-remove.yaml"
+        file = open(filename, "w")
+        file.write("{key1: { child1: 123, child2: abc }, key2: false, key3: [a,b,c]}")
+        file.close()
+
+        soyaml.append([filename, "key3", "d"])
+
+        file = open(filename, "r")
+        actual = file.read()
+        file.close()
+        expected = "key1:\n  child1: 123\n  child2: abc\nkey2: false\nkey3:\n- a\n- b\n- c\n- d\n"
+        self.assertEqual(actual, expected)
+
+    def test_append_nested(self):
+        filename = "/tmp/so-yaml_test-remove.yaml"
+        file = open(filename, "w")
+        file.write("{key1: { child1: 123, child2: [a,b,c] }, key2: false, key3: [e,f,g]}")
+        file.close()
+
+        soyaml.append([filename, "key1.child2", "d"])
+
+        file = open(filename, "r")
+        actual = file.read()
+        file.close()
+
+        expected = "key1:\n  child1: 123\n  child2:\n  - a\n  - b\n  - c\n  - d\nkey2: false\nkey3:\n- e\n- f\n- g\n"
+        self.assertEqual(actual, expected)
+
+    def test_append_nested_deep(self):
+        filename = "/tmp/so-yaml_test-remove.yaml"
+        file = open(filename, "w")
+        file.write("{key1: { child1: 123, child2: { deep1: 45, deep2: [a,b,c] } }, key2: false, key3: [e,f,g]}")
+        file.close()
+
+        soyaml.append([filename, "key1.child2.deep2", "d"])
+
+        file = open(filename, "r")
+        actual = file.read()
+        file.close()
+
+        expected = "key1:\n  child1: 123\n  child2:\n    deep1: 45\n    deep2:\n    - a\n    - b\n    - c\n    - d\nkey2: false\nkey3:\n- e\n- f\n- g\n"
+        self.assertEqual(actual, expected)
+
+    def test_append_key_noexist(self):
+        filename = "/tmp/so-yaml_test-append.yaml"
+        file = open(filename, "w")
+        file.write("{key1: { child1: 123, child2: { deep1: 45, deep2: [a,b,c] } }, key2: false, key3: [e,f,g]}")
+        file.close()
+
+        with patch('sys.exit', new=MagicMock()) as sysmock:
+            with patch('sys.stdout', new=StringIO()) as mock_stdout:
+                sys.argv = ["cmd", "append", filename, "key4", "h"]
+                soyaml.main()
+                sysmock.assert_called()
+                self.assertEqual(mock_stdout.getvalue(), "The key provided does not exist. No action was taken on the file.\n")
+
+    def test_append_key_noexist_deep(self):
+        filename = "/tmp/so-yaml_test-append.yaml"
+        file = open(filename, "w")
+        file.write("{key1: { child1: 123, child2: { deep1: 45, deep2: [a,b,c] } }, key2: false, key3: [e,f,g]}")
+        file.close()
+
+        with patch('sys.exit', new=MagicMock()) as sysmock:
+            with patch('sys.stdout', new=StringIO()) as mock_stdout:
+                sys.argv = ["cmd", "append", filename, "key1.child2.deep3", "h"]
+                soyaml.main()
+                sysmock.assert_called()
+                self.assertEqual(mock_stdout.getvalue(), "The key provided does not exist. No action was taken on the file.\n")
+
+    def test_append_key_nonlist(self):
+        filename = "/tmp/so-yaml_test-append.yaml"
+        file = open(filename, "w")
+        file.write("{key1: { child1: 123, child2: { deep1: 45, deep2: [a,b,c] } }, key2: false, key3: [e,f,g]}")
+        file.close()
+
+        with patch('sys.exit', new=MagicMock()) as sysmock:
+            with patch('sys.stdout', new=StringIO()) as mock_stdout:
+                sys.argv = ["cmd", "append", filename, "key1", "h"]
+                soyaml.main()
+                sysmock.assert_called()
+                self.assertEqual(mock_stdout.getvalue(), "The existing value for the given key is not a list. No action was taken on the file.\n")
+
+    def test_append_key_nonlist_deep(self):
+        filename = "/tmp/so-yaml_test-append.yaml"
+        file = open(filename, "w")
+        file.write("{key1: { child1: 123, child2: { deep1: 45, deep2: [a,b,c] } }, key2: false, key3: [e,f,g]}")
+        file.close()
+
+        with patch('sys.exit', new=MagicMock()) as sysmock:
+            with patch('sys.stdout', new=StringIO()) as mock_stdout:
+                sys.argv = ["cmd", "append", filename, "key1.child2.deep1", "h"]
+                soyaml.main()
+                sysmock.assert_called()
+                self.assertEqual(mock_stdout.getvalue(), "The existing value for the given key is not a list. No action was taken on the file.\n")

salt/manager/tools/sbin/soup

@@ -594,6 +594,19 @@ up_to_2.4.50() {
   touch /opt/so/saltstack/local/pillar/stig/adv_stig.sls
   touch /opt/so/saltstack/local/pillar/stig/soc_stig.sls
 
+  # the file_roots need to be update due to salt 3006.6 upgrade not allowing symlinks outside the file_roots
+  # put new so-yaml in place
+  echo "Updating so-yaml"
+  \cp -v "$UPDATE_DIR/salt/manager/tools/sbin/so-yaml.py" "$DEFAULT_SALT_DIR/salt/manager/tools/sbin/"
+  \cp -v "$UPDATE_DIR/salt/manager/tools/sbin/so-yaml.py" /usr/sbin/
+  echo "Creating a backup of the salt-master config."
+  # INSTALLEDVERSION is 2.4.40 at this point, but we want the backup to have the version
+  # so was at prior to starting upgrade. use POSTVERSION here since it doesnt change until
+  # post upgrade changes. POSTVERSION set to INSTALLEDVERSION at start of soup
+  cp -v /etc/salt/master "/etc/salt/master.so-$POSTVERSION.bak"
+  echo "Adding /opt/so/rules to file_roots in /etc/salt/master using so-yaml"
+  so-yaml.py append /etc/salt/master file_roots.base /opt/so/rules
+
   INSTALLEDVERSION=2.4.50
 }
 
@@ -937,9 +950,6 @@ main() {
 
     systemctl_func "stop" "$cron_service_name"
 
-    # update mine items prior to stopping salt-minion and salt-master
-    update_salt_mine
-
     echo "Updating dockers to $NEWVERSION."
     if [[ $is_airgap -eq 0 ]]; then
       airgap_update_dockers
@@ -1015,6 +1025,9 @@ main() {
     salt-call state.apply salt.minion -l info queue=True
     echo ""
 
+    # ensure the mine is updated and populated before highstates run, following the salt-master restart
+    update_salt_mine
+
     enable_highstate
 
     echo ""

salt/salt/master.defaults.yaml

@@ -1,4 +1,4 @@
 # version cannot be used elsewhere in this pillar as soup is grepping for it to determine if Salt needs to be patched
 salt:
   master:
-    version: 3006.5
+    version: 3006.6

salt/salt/minion.defaults.yaml

@@ -1,6 +1,6 @@
 # version cannot be used elsewhere in this pillar as soup is grepping for it to determine if Salt needs to be patched
 salt:
   minion:
-    version: 3006.5
+    version: 3006.6
     check_threshold: 3600 # in seconds, threshold used for so-salt-minion-check. any value less than 600 seconds may cause a lot of salt-minion restarts since the job to touch the file occurs every 5-8 minutes by default
     service_start_delay: 30 # in seconds.

salt/suricata/config.sls

@@ -84,10 +84,12 @@ suridatadir:
     - mode: 770
     - makedirs: True
 
+# salt:// would resolve to /opt/so/rules because of the defined file_roots and
+# nids not existing under /opt/so/saltstack/local/salt or /opt/so/saltstack/default/salt
 surirulesync:
   file.recurse:
     - name: /opt/so/conf/suricata/rules/
-    - source: salt://suricata/rules/
+    - source: salt://nids/
     - user: 940
     - group: 940
     - show_changes: False