From 0d5db58c86f39521ec954d186db6050b7d41fbad Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 1 Feb 2024 10:32:41 -0500 Subject: [PATCH 1/8] upgrade salt3006.6 --- salt/salt/master.defaults.yaml | 2 +- salt/salt/minion.defaults.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/salt/master.defaults.yaml b/salt/salt/master.defaults.yaml index 1b4d2e63a..19677f70b 100644 --- a/salt/salt/master.defaults.yaml +++ b/salt/salt/master.defaults.yaml @@ -1,4 +1,4 @@ # version cannot be used elsewhere in this pillar as soup is grepping for it to determine if Salt needs to be patched salt: master: - version: 3006.5 + version: 3006.6 diff --git a/salt/salt/minion.defaults.yaml b/salt/salt/minion.defaults.yaml index c15929951..2e4ebc93e 100644 --- a/salt/salt/minion.defaults.yaml +++ b/salt/salt/minion.defaults.yaml @@ -1,6 +1,6 @@ # version cannot be used elsewhere in this pillar as soup is grepping for it to determine if Salt needs to be patched salt: minion: - version: 3006.5 + version: 3006.6 check_threshold: 3600 # in seconds, threshold used for so-salt-minion-check. any value less than 600 seconds may cause a lot of salt-minion restarts since the job to touch the file occurs every 5-8 minutes by default service_start_delay: 30 # in seconds. From 2643ae08a75c097abc1de60397f33b72a66655f0 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 5 Feb 2024 17:54:30 -0500 Subject: [PATCH 2/8] add append to list --- salt/manager/tools/sbin/so-yaml.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/salt/manager/tools/sbin/so-yaml.py b/salt/manager/tools/sbin/so-yaml.py index 874fc9e0f..572585df9 100755 --- a/salt/manager/tools/sbin/so-yaml.py +++ b/salt/manager/tools/sbin/so-yaml.py @@ -16,12 +16,14 @@ lockFile = "/tmp/so-yaml.lock" def showUsage(args): print('Usage: {} [ARGS...]'.format(sys.argv[0])) print(' General commands:') + print(' append - Append a list item to a yaml key, if it exists and is a list. Requires KEY and LISTITEM args.') print(' remove - Removes a yaml key, if it exists. Requires KEY arg.') print(' help - Prints this usage information.') print('') print(' Where:') print(' YAML_FILE - Path to the file that will be modified. Ex: /opt/so/conf/service/conf.yaml') print(' KEY - YAML key, does not support \' or " characters at this time. Ex: level1.level2') + print(' LISTITEM - Item to add to the list.') sys.exit(1) @@ -35,6 +37,32 @@ def writeYaml(filename, content): file = open(filename, "w") return yaml.dump(content, file) +def appendItem(content, key, listItem): + pieces = key.split(".", 1) + if len(pieces) > 1: + appendItem(content[pieces[0]], pieces[1], listItem) + else: + try: + content[key].append(listItem) + except TypeError: + print("The contents key provided is likely not a list. No action was taken on the file.") + return 1 + +def append(args): + if len(args) != 3: + print('Missing filename, key arg, or list item to append', file=sys.stderr) + showUsage(None) + return + + filename = args[0] + key = args[1] + listItem = args[2] + + content = loadYaml(filename) + appendItem(content, key, listItem) + writeYaml(filename, content) + + return 0 def removeKey(content, key): pieces = key.split(".", 1) @@ -69,6 +97,7 @@ def main(): commands = { "help": showUsage, + "append": append, "remove": remove, } From 9d62ade32e72f10e058428c1e441b0de225d8527 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 6 Feb 2024 11:14:27 -0500 Subject: [PATCH 3/8] update so-yaml tests --- salt/manager/tools/sbin/so-yaml.py | 2 +- salt/manager/tools/sbin/so-yaml_test.py | 44 +++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/so-yaml.py b/salt/manager/tools/sbin/so-yaml.py index 572585df9..9f4bdbb98 100755 --- a/salt/manager/tools/sbin/so-yaml.py +++ b/salt/manager/tools/sbin/so-yaml.py @@ -45,7 +45,7 @@ def appendItem(content, key, listItem): try: content[key].append(listItem) except TypeError: - print("The contents key provided is likely not a list. No action was taken on the file.") + print("The key provided is likely not a list. No action was taken on the file.") return 1 def append(args): diff --git a/salt/manager/tools/sbin/so-yaml_test.py b/salt/manager/tools/sbin/so-yaml_test.py index 7d0ed1a8e..f70314ba3 100644 --- a/salt/manager/tools/sbin/so-yaml_test.py +++ b/salt/manager/tools/sbin/so-yaml_test.py @@ -105,3 +105,47 @@ class TestRemove(unittest.TestCase): self.assertEqual(actual, expected) sysmock.assert_called_once_with(1) self.assertIn(mock_stdout.getvalue(), "Missing filename or key arg\n") + + def test_append(self): + filename = "/tmp/so-yaml_test-remove.yaml" + file = open(filename, "w") + file.write("{key1: { child1: 123, child2: abc }, key2: false, key3: [a,b,c]}") + file.close() + + soyaml.append([filename, "key3", "d"]) + + file = open(filename, "r") + actual = file.read() + file.close() + expected = "key1:\n child1: 123\n child2: abc\nkey2: false\nkey3:\n- a\n- b\n- c\n- d\n" + self.assertEqual(actual, expected) + + def test_append_nested(self): + filename = "/tmp/so-yaml_test-remove.yaml" + file = open(filename, "w") + file.write("{key1: { child1: 123, child2: [a,b,c] }, key2: false, key3: [e,f,g]}") + file.close() + + soyaml.append([filename, "key1.child2", "d"]) + + file = open(filename, "r") + actual = file.read() + file.close() + + expected = "key1:\n child1: 123\n child2:\n - a\n - b\n - c\n - d\nkey2: false\nkey3:\n- e\n- f\n- g\n" + self.assertEqual(actual, expected) + + def test_append_nested_deep(self): + filename = "/tmp/so-yaml_test-remove.yaml" + file = open(filename, "w") + file.write("{key1: { child1: 123, child2: { deep1: 45, deep2: [a,b,c] } }, key2: false, key3: [e,f,g]}") + file.close() + + soyaml.append([filename, "key1.child2.deep2", "d"]) + + file = open(filename, "r") + actual = file.read() + file.close() + + expected = "key1:\n child1: 123\n child2:\n deep1: 45\n deep2:\n - a\n - b\n - c\n - d\nkey2: false\nkey3:\n- e\n- f\n- g\n" + self.assertEqual(actual, expected) From b3f61536670050996fd539444863aa95a203a2b9 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 6 Feb 2024 16:15:54 -0500 Subject: [PATCH 4/8] update so-yaml tests --- salt/manager/tools/sbin/so-yaml.py | 7 +++- salt/manager/tools/sbin/so-yaml_test.py | 52 +++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/salt/manager/tools/sbin/so-yaml.py b/salt/manager/tools/sbin/so-yaml.py index 9f4bdbb98..24d520891 100755 --- a/salt/manager/tools/sbin/so-yaml.py +++ b/salt/manager/tools/sbin/so-yaml.py @@ -44,8 +44,11 @@ def appendItem(content, key, listItem): else: try: content[key].append(listItem) - except TypeError: - print("The key provided is likely not a list. No action was taken on the file.") + except AttributeError: + print("The key provided is not a list. No action was taken on the file.") + return 1 + except KeyError: + print("The key provided does not exist. No action was taken on the file.") return 1 def append(args): diff --git a/salt/manager/tools/sbin/so-yaml_test.py b/salt/manager/tools/sbin/so-yaml_test.py index f70314ba3..59db43860 100644 --- a/salt/manager/tools/sbin/so-yaml_test.py +++ b/salt/manager/tools/sbin/so-yaml_test.py @@ -149,3 +149,55 @@ class TestRemove(unittest.TestCase): expected = "key1:\n child1: 123\n child2:\n deep1: 45\n deep2:\n - a\n - b\n - c\n - d\nkey2: false\nkey3:\n- e\n- f\n- g\n" self.assertEqual(actual, expected) + + def test_append_key_noexist(self): + filename = "/tmp/so-yaml_test-append.yaml" + file = open(filename, "w") + file.write("{key1: { child1: 123, child2: { deep1: 45, deep2: [a,b,c] } }, key2: false, key3: [e,f,g]}") + file.close() + + with patch('sys.exit', new=MagicMock()) as sysmock: + with patch('sys.stdout', new=StringIO()) as mock_stdout: + sys.argv = ["cmd", "append", filename, "key4", "h"] + soyaml.main() + sysmock.assert_called() + self.assertEqual(mock_stdout.getvalue(), "The key provided does not exist. No action was taken on the file.\n") + + def test_append_key_noexist_deep(self): + filename = "/tmp/so-yaml_test-append.yaml" + file = open(filename, "w") + file.write("{key1: { child1: 123, child2: { deep1: 45, deep2: [a,b,c] } }, key2: false, key3: [e,f,g]}") + file.close() + + with patch('sys.exit', new=MagicMock()) as sysmock: + with patch('sys.stdout', new=StringIO()) as mock_stdout: + sys.argv = ["cmd", "append", filename, "key1.child2.deep3", "h"] + soyaml.main() + sysmock.assert_called() + self.assertEqual(mock_stdout.getvalue(), "The key provided does not exist. No action was taken on the file.\n") + + def test_append_key_nonlist(self): + filename = "/tmp/so-yaml_test-append.yaml" + file = open(filename, "w") + file.write("{key1: { child1: 123, child2: { deep1: 45, deep2: [a,b,c] } }, key2: false, key3: [e,f,g]}") + file.close() + + with patch('sys.exit', new=MagicMock()) as sysmock: + with patch('sys.stdout', new=StringIO()) as mock_stdout: + sys.argv = ["cmd", "append", filename, "key1", "h"] + soyaml.main() + sysmock.assert_called() + self.assertEqual(mock_stdout.getvalue(), "The key provided is not a list. No action was taken on the file.\n") + + def test_append_key_nonlist_deep(self): + filename = "/tmp/so-yaml_test-append.yaml" + file = open(filename, "w") + file.write("{key1: { child1: 123, child2: { deep1: 45, deep2: [a,b,c] } }, key2: false, key3: [e,f,g]}") + file.close() + + with patch('sys.exit', new=MagicMock()) as sysmock: + with patch('sys.stdout', new=StringIO()) as mock_stdout: + sys.argv = ["cmd", "append", filename, "key1.child2.deep1", "h"] + soyaml.main() + sysmock.assert_called() + self.assertEqual(mock_stdout.getvalue(), "The key provided is not a list. No action was taken on the file.\n") From 24fd3ef8cc40a4b0f3e7083ae935171079de11f8 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 6 Feb 2024 16:22:13 -0500 Subject: [PATCH 5/8] uopdate error message --- salt/manager/tools/sbin/so-yaml.py | 2 +- salt/manager/tools/sbin/so-yaml_test.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/manager/tools/sbin/so-yaml.py b/salt/manager/tools/sbin/so-yaml.py index 24d520891..41cab0b23 100755 --- a/salt/manager/tools/sbin/so-yaml.py +++ b/salt/manager/tools/sbin/so-yaml.py @@ -45,7 +45,7 @@ def appendItem(content, key, listItem): try: content[key].append(listItem) except AttributeError: - print("The key provided is not a list. No action was taken on the file.") + print("The existing value for the given key is not a list. No action was taken on the file.") return 1 except KeyError: print("The key provided does not exist. No action was taken on the file.") diff --git a/salt/manager/tools/sbin/so-yaml_test.py b/salt/manager/tools/sbin/so-yaml_test.py index 59db43860..488877ea1 100644 --- a/salt/manager/tools/sbin/so-yaml_test.py +++ b/salt/manager/tools/sbin/so-yaml_test.py @@ -187,7 +187,7 @@ class TestRemove(unittest.TestCase): sys.argv = ["cmd", "append", filename, "key1", "h"] soyaml.main() sysmock.assert_called() - self.assertEqual(mock_stdout.getvalue(), "The key provided is not a list. No action was taken on the file.\n") + self.assertEqual(mock_stdout.getvalue(), "The existing value for the given key is not a list. No action was taken on the file.\n") def test_append_key_nonlist_deep(self): filename = "/tmp/so-yaml_test-append.yaml" @@ -200,4 +200,4 @@ class TestRemove(unittest.TestCase): sys.argv = ["cmd", "append", filename, "key1.child2.deep1", "h"] soyaml.main() sysmock.assert_called() - self.assertEqual(mock_stdout.getvalue(), "The key provided is not a list. No action was taken on the file.\n") + self.assertEqual(mock_stdout.getvalue(), "The existing value for the given key is not a list. No action was taken on the file.\n") From f97d0f2f3644b298ef14c313fd9e94c39baa9573 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 7 Feb 2024 09:25:56 -0500 Subject: [PATCH 6/8] add /opt/so/rules/ to files_roots --- files/salt/master/master | 1 + salt/manager/tools/sbin/soup | 10 ++++++++++ salt/suricata/config.sls | 4 +++- 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/files/salt/master/master b/files/salt/master/master index b93fa93de..e309a560b 100644 --- a/files/salt/master/master +++ b/files/salt/master/master @@ -41,6 +41,7 @@ file_roots: base: - /opt/so/saltstack/local/salt - /opt/so/saltstack/default/salt + - /opt/so/rules # The master_roots setting configures a master-only copy of the file_roots dictionary, diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index a250116d1..6f086469a 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -594,6 +594,16 @@ up_to_2.4.50() { touch /opt/so/saltstack/local/pillar/stig/adv_stig.sls touch /opt/so/saltstack/local/pillar/stig/soc_stig.sls + # the file_roots need to be update due to salt 3006.6 upgrade not allowing symlinks outside the file_roots + # put new so-yaml in place + echo "Updating so-yaml" + \cp -v "$UPDATE_DIR/salt/manager/tools/sbin/so-yaml.py" "$DEFAULT_SALT_DIR/salt/manager/tools/sbin/" + \cp -v "$UPDATE_DIR/salt/manager/tools/sbin/so-yaml.py" /usr/sbin/ + echo "Creating a backup of the salt-master config." + cp -v /etc/salt/master "/etc/salt/master.so-$INSTALLEDVERSION" + echo "Adding /opt/so/rules to file_roots using so-yaml" + so-yaml.py append /etc/salt/master file_roots.base /opt/so/rules + INSTALLEDVERSION=2.4.50 } diff --git a/salt/suricata/config.sls b/salt/suricata/config.sls index 8d5279349..4804565ce 100644 --- a/salt/suricata/config.sls +++ b/salt/suricata/config.sls @@ -84,10 +84,12 @@ suridatadir: - mode: 770 - makedirs: True +# salt:// would resolve to /opt/so/rules because of the defined file_roots and +# nids not existing under /opt/so/saltstack/local/salt or /opt/so/saltstack/default/salt surirulesync: file.recurse: - name: /opt/so/conf/suricata/rules/ - - source: salt://suricata/rules/ + - source: salt://nids/ - user: 940 - group: 940 - show_changes: False From e42e07b245d32dd2bdb1d54e807d478209506efa Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 7 Feb 2024 13:05:45 -0500 Subject: [PATCH 7/8] update salt mine after salt-master restarts --- salt/manager/tools/sbin/soup | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 6f086469a..b4647bdbc 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -947,9 +947,6 @@ main() { systemctl_func "stop" "$cron_service_name" - # update mine items prior to stopping salt-minion and salt-master - update_salt_mine - echo "Updating dockers to $NEWVERSION." if [[ $is_airgap -eq 0 ]]; then airgap_update_dockers @@ -1025,6 +1022,9 @@ main() { salt-call state.apply salt.minion -l info queue=True echo "" + # ensure the mine is updated and populated before highstates run, following the salt-master restart + update_salt_mine + enable_highstate echo "" From 6534f392a9334a2dd1f1b89b9961d116d5c44de7 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 7 Feb 2024 14:25:28 -0500 Subject: [PATCH 8/8] update backup filename --- salt/manager/tools/sbin/soup | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index b4647bdbc..5bade9891 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -600,8 +600,11 @@ up_to_2.4.50() { \cp -v "$UPDATE_DIR/salt/manager/tools/sbin/so-yaml.py" "$DEFAULT_SALT_DIR/salt/manager/tools/sbin/" \cp -v "$UPDATE_DIR/salt/manager/tools/sbin/so-yaml.py" /usr/sbin/ echo "Creating a backup of the salt-master config." - cp -v /etc/salt/master "/etc/salt/master.so-$INSTALLEDVERSION" - echo "Adding /opt/so/rules to file_roots using so-yaml" + # INSTALLEDVERSION is 2.4.40 at this point, but we want the backup to have the version + # so was at prior to starting upgrade. use POSTVERSION here since it doesnt change until + # post upgrade changes. POSTVERSION set to INSTALLEDVERSION at start of soup + cp -v /etc/salt/master "/etc/salt/master.so-$POSTVERSION.bak" + echo "Adding /opt/so/rules to file_roots in /etc/salt/master using so-yaml" so-yaml.py append /etc/salt/master file_roots.base /opt/so/rules INSTALLEDVERSION=2.4.50