diff --git a/pyci.sh b/pyci.sh index e85287063..8cbee5e75 100755 --- a/pyci.sh +++ b/pyci.sh @@ -15,12 +15,16 @@ TARGET_DIR=${1:-.} PATH=$PATH:/usr/local/bin -if ! which pytest &> /dev/null || ! which flake8 &> /dev/null ; then - echo "Missing dependencies. Consider running the following command:" - echo " python -m pip install flake8 pytest pytest-cov" +if [ ! -d .venv ]; then + python -m venv .venv +fi + +source .venv/bin/activate + +if ! pip install flake8 pytest pytest-cov pyyaml; then + echo "Unable to install dependencies." exit 1 fi -pip install pytest pytest-cov flake8 "$TARGET_DIR" "--config=${HOME_DIR}/pytest.ini" -python3 -m pytest "--cov-config=${HOME_DIR}/pytest.ini" "--cov=$TARGET_DIR" --doctest-modules --cov-report=term --cov-fail-under=100 "$TARGET_DIR" \ No newline at end of file +python3 -m pytest "--cov-config=${HOME_DIR}/pytest.ini" "--cov=$TARGET_DIR" --doctest-modules --cov-report=term --cov-fail-under=100 "$TARGET_DIR" diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index 8b563ef1d..da1a6d2a2 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -112,8 +112,8 @@ function testMinion() { result=$? # If this so-minion script is not running on the given minion ID, run so-test remotely on the sensor as well - local_id=$(lookup_grain id) - if [[ ! "$local_id" =~ "${MINION_ID}_" ]]; then + local_id=$(lookup_grain id) + if [[ ! "$local_id" =~ "${MINION_ID}_" && "$local_id" != "${MINION_ID}" ]]; then salt "$MINION_ID" cmd.run 'so-test' result=$? fi diff --git a/salt/manager/tools/sbin/so-yaml.py b/salt/manager/tools/sbin/so-yaml.py index 5427a2e48..cddc827b5 100755 --- a/salt/manager/tools/sbin/so-yaml.py +++ b/salt/manager/tools/sbin/so-yaml.py @@ -14,19 +14,20 @@ lockFile = "/tmp/so-yaml.lock" def showUsage(args): - print('Usage: {} [ARGS...]'.format(sys.argv[0])) - print(' General commands:') - print(' append - Append a list item to a yaml key, if it exists and is a list. Requires KEY and LISTITEM args.') - print(' add - Add a new key and set its value. Fails if key already exists. Requires KEY and VALUE args.') - print(' remove - Removes a yaml key, if it exists. Requires KEY arg.') - print(' replace - Replaces (or adds) a new key and set its value. Requires KEY and VALUE args.') - print(' help - Prints this usage information.') - print('') - print(' Where:') - print(' YAML_FILE - Path to the file that will be modified. Ex: /opt/so/conf/service/conf.yaml') - print(' KEY - YAML key, does not support \' or " characters at this time. Ex: level1.level2') - print(' VALUE - Value to set for a given key') - print(' LISTITEM - Item to append to a given key\'s list value') + print('Usage: {} [ARGS...]'.format(sys.argv[0]), file=sys.stderr) + print(' General commands:', file=sys.stderr) + print(' append - Append a list item to a yaml key, if it exists and is a list. Requires KEY and LISTITEM args.', file=sys.stderr) + print(' add - Add a new key and set its value. Fails if key already exists. Requires KEY and VALUE args.', file=sys.stderr) + print(' get - Displays (to stdout) the value stored in the given key. Requires KEY arg.', file=sys.stderr) + print(' remove - Removes a yaml key, if it exists. Requires KEY arg.', file=sys.stderr) + print(' replace - Replaces (or adds) a new key and set its value. Requires KEY and VALUE args.', file=sys.stderr) + print(' help - Prints this usage information.', file=sys.stderr) + print('', file=sys.stderr) + print(' Where:', file=sys.stderr) + print(' YAML_FILE - Path to the file that will be modified. Ex: /opt/so/conf/service/conf.yaml', file=sys.stderr) + print(' KEY - YAML key, does not support \' or " characters at this time. Ex: level1.level2', file=sys.stderr) + print(' VALUE - Value to set for a given key', file=sys.stderr) + print(' LISTITEM - Item to append to a given key\'s list value', file=sys.stderr) sys.exit(1) @@ -38,7 +39,7 @@ def loadYaml(filename): def writeYaml(filename, content): file = open(filename, "w") - return yaml.dump(content, file) + return yaml.safe_dump(content, file) def appendItem(content, key, listItem): @@ -49,15 +50,15 @@ def appendItem(content, key, listItem): try: content[key].append(listItem) except AttributeError: - print("The existing value for the given key is not a list. No action was taken on the file.") + print("The existing value for the given key is not a list. No action was taken on the file.", file=sys.stderr) return 1 except KeyError: - print("The key provided does not exist. No action was taken on the file.") + print("The key provided does not exist. No action was taken on the file.", file=sys.stderr) return 1 def convertType(value): - if len(value) > 0 and (not value.startswith("0") or len(value) == 1): + if isinstance(value, str) and len(value) > 0 and (not value.startswith("0") or len(value) == 1): if "." in value: try: value = float(value) @@ -83,7 +84,7 @@ def append(args): if len(args) != 3: print('Missing filename, key arg, or list item to append', file=sys.stderr) showUsage(None) - return + return 1 filename = args[0] key = args[1] @@ -112,7 +113,7 @@ def add(args): if len(args) != 3: print('Missing filename, key arg, and/or value', file=sys.stderr) showUsage(None) - return + return 1 filename = args[0] key = args[1] @@ -137,7 +138,7 @@ def remove(args): if len(args) != 2: print('Missing filename or key arg', file=sys.stderr) showUsage(None) - return + return 1 filename = args[0] key = args[1] @@ -153,7 +154,7 @@ def replace(args): if len(args) != 3: print('Missing filename, key arg, and/or value', file=sys.stderr) showUsage(None) - return + return 1 filename = args[0] key = args[1] @@ -167,6 +168,32 @@ def replace(args): return 0 +def getKeyValue(content, key): + pieces = key.split(".", 1) + if len(pieces) > 1: + return getKeyValue(content[pieces[0]], pieces[1]) + return content.get(key, None) + + +def get(args): + if len(args) != 2: + print('Missing filename or key arg', file=sys.stderr) + showUsage(None) + return 1 + + filename = args[0] + key = args[1] + + content = loadYaml(filename) + output = getKeyValue(content, key) + if output is None: + print("Not found", file=sys.stderr) + return 2 + + print(yaml.safe_dump(output)) + return 0 + + def main(): args = sys.argv[1:] @@ -178,6 +205,7 @@ def main(): "help": showUsage, "add": add, "append": append, + "get": get, "remove": remove, "replace": replace, } @@ -195,11 +223,11 @@ def main(): break except Exception: if lockAttempts == 1: - print("Waiting for lock file to be released from another process...") + print("Waiting for lock file to be released from another process...", file=sys.stderr) time.sleep(2) if lockAttempts == maxAttempts: - print("Lock file (" + lockFile + ") could not be created; proceeding without lock.") + print("Lock file (" + lockFile + ") could not be created; proceeding without lock.", file=sys.stderr) cmd = commands.get(args[0], showUsage) code = cmd(args[1:]) diff --git a/salt/manager/tools/sbin/so-yaml_test.py b/salt/manager/tools/sbin/so-yaml_test.py index 7effabac9..ca9839e02 100644 --- a/salt/manager/tools/sbin/so-yaml_test.py +++ b/salt/manager/tools/sbin/so-yaml_test.py @@ -15,40 +15,40 @@ class TestRemove(unittest.TestCase): def test_main_missing_input(self): with patch('sys.exit', new=MagicMock()) as sysmock: - with patch('sys.stderr', new=StringIO()) as mock_stdout: + with patch('sys.stderr', new=StringIO()) as mock_stderr: sys.argv = ["cmd"] soyaml.main() sysmock.assert_called_once_with(1) - self.assertIn(mock_stdout.getvalue(), "Usage:") + self.assertIn("Usage:", mock_stderr.getvalue()) def test_main_help_locked(self): filename = "/tmp/so-yaml.lock" file = open(filename, "w") file.write = "fake lock file" with patch('sys.exit', new=MagicMock()) as sysmock: - with patch('sys.stderr', new=StringIO()) as mock_stdout: + with patch('sys.stderr', new=StringIO()) as mock_stderr: with patch('time.sleep', new=MagicMock()) as mock_sleep: sys.argv = ["cmd", "help"] soyaml.main() sysmock.assert_called() mock_sleep.assert_called_with(2) - self.assertIn(mock_stdout.getvalue(), "Usage:") + self.assertIn("Usage:", mock_stderr.getvalue()) def test_main_help(self): with patch('sys.exit', new=MagicMock()) as sysmock: - with patch('sys.stderr', new=StringIO()) as mock_stdout: + with patch('sys.stderr', new=StringIO()) as mock_stderr: sys.argv = ["cmd", "help"] soyaml.main() sysmock.assert_called() - self.assertIn(mock_stdout.getvalue(), "Usage:") + self.assertIn("Usage:", mock_stderr.getvalue()) def test_remove_missing_arg(self): with patch('sys.exit', new=MagicMock()) as sysmock: - with patch('sys.stderr', new=StringIO()) as mock_stdout: + with patch('sys.stderr', new=StringIO()) as mock_stderr: sys.argv = ["cmd", "help"] soyaml.remove(["file"]) sysmock.assert_called() - self.assertIn(mock_stdout.getvalue(), "Missing filename or key arg\n") + self.assertIn("Missing filename or key arg\n", mock_stderr.getvalue()) def test_remove(self): filename = "/tmp/so-yaml_test-remove.yaml" @@ -97,7 +97,7 @@ class TestRemove(unittest.TestCase): def test_remove_missing_args(self): with patch('sys.exit', new=MagicMock()) as sysmock: - with patch('sys.stderr', new=StringIO()) as mock_stdout: + with patch('sys.stderr', new=StringIO()) as mock_stderr: filename = "/tmp/so-yaml_test-remove.yaml" file = open(filename, "w") file.write("{key1: { child1: 123, child2: abc }, key2: false}") @@ -112,15 +112,15 @@ class TestRemove(unittest.TestCase): expected = "{key1: { child1: 123, child2: abc }, key2: false}" self.assertEqual(actual, expected) sysmock.assert_called_once_with(1) - self.assertIn(mock_stdout.getvalue(), "Missing filename or key arg\n") + self.assertIn("Missing filename or key arg\n", mock_stderr.getvalue()) def test_append_missing_arg(self): with patch('sys.exit', new=MagicMock()) as sysmock: - with patch('sys.stderr', new=StringIO()) as mock_stdout: + with patch('sys.stderr', new=StringIO()) as mock_stderr: sys.argv = ["cmd", "help"] soyaml.append(["file", "key"]) sysmock.assert_called() - self.assertIn(mock_stdout.getvalue(), "Missing filename, key arg, or list item to append\n") + self.assertIn("Missing filename, key arg, or list item to append\n", mock_stderr.getvalue()) def test_append(self): filename = "/tmp/so-yaml_test-remove.yaml" @@ -173,11 +173,11 @@ class TestRemove(unittest.TestCase): file.close() with patch('sys.exit', new=MagicMock()) as sysmock: - with patch('sys.stdout', new=StringIO()) as mock_stdout: + with patch('sys.stderr', new=StringIO()) as mock_stderr: sys.argv = ["cmd", "append", filename, "key4", "h"] soyaml.main() sysmock.assert_called() - self.assertEqual(mock_stdout.getvalue(), "The key provided does not exist. No action was taken on the file.\n") + self.assertEqual("The key provided does not exist. No action was taken on the file.\n", mock_stderr.getvalue()) def test_append_key_noexist_deep(self): filename = "/tmp/so-yaml_test-append.yaml" @@ -186,11 +186,11 @@ class TestRemove(unittest.TestCase): file.close() with patch('sys.exit', new=MagicMock()) as sysmock: - with patch('sys.stdout', new=StringIO()) as mock_stdout: + with patch('sys.stderr', new=StringIO()) as mock_stderr: sys.argv = ["cmd", "append", filename, "key1.child2.deep3", "h"] soyaml.main() sysmock.assert_called() - self.assertEqual(mock_stdout.getvalue(), "The key provided does not exist. No action was taken on the file.\n") + self.assertEqual("The key provided does not exist. No action was taken on the file.\n", mock_stderr.getvalue()) def test_append_key_nonlist(self): filename = "/tmp/so-yaml_test-append.yaml" @@ -199,11 +199,11 @@ class TestRemove(unittest.TestCase): file.close() with patch('sys.exit', new=MagicMock()) as sysmock: - with patch('sys.stdout', new=StringIO()) as mock_stdout: + with patch('sys.stderr', new=StringIO()) as mock_stderr: sys.argv = ["cmd", "append", filename, "key1", "h"] soyaml.main() sysmock.assert_called() - self.assertEqual(mock_stdout.getvalue(), "The existing value for the given key is not a list. No action was taken on the file.\n") + self.assertEqual("The existing value for the given key is not a list. No action was taken on the file.\n", mock_stderr.getvalue()) def test_append_key_nonlist_deep(self): filename = "/tmp/so-yaml_test-append.yaml" @@ -212,11 +212,11 @@ class TestRemove(unittest.TestCase): file.close() with patch('sys.exit', new=MagicMock()) as sysmock: - with patch('sys.stdout', new=StringIO()) as mock_stdout: + with patch('sys.stderr', new=StringIO()) as mock_stderr: sys.argv = ["cmd", "append", filename, "key1.child2.deep1", "h"] soyaml.main() sysmock.assert_called() - self.assertEqual(mock_stdout.getvalue(), "The existing value for the given key is not a list. No action was taken on the file.\n") + self.assertEqual("The existing value for the given key is not a list. No action was taken on the file.\n", mock_stderr.getvalue()) def test_add_key(self): content = {} @@ -244,11 +244,11 @@ class TestRemove(unittest.TestCase): def test_add_missing_arg(self): with patch('sys.exit', new=MagicMock()) as sysmock: - with patch('sys.stderr', new=StringIO()) as mock_stdout: + with patch('sys.stderr', new=StringIO()) as mock_stderr: sys.argv = ["cmd", "help"] soyaml.add(["file", "key"]) sysmock.assert_called() - self.assertIn(mock_stdout.getvalue(), "Missing filename, key arg, and/or value\n") + self.assertIn("Missing filename, key arg, and/or value\n", mock_stderr.getvalue()) def test_add(self): filename = "/tmp/so-yaml_test-add.yaml" @@ -296,11 +296,11 @@ class TestRemove(unittest.TestCase): def test_replace_missing_arg(self): with patch('sys.exit', new=MagicMock()) as sysmock: - with patch('sys.stderr', new=StringIO()) as mock_stdout: + with patch('sys.stderr', new=StringIO()) as mock_stderr: sys.argv = ["cmd", "help"] soyaml.replace(["file", "key"]) sysmock.assert_called() - self.assertIn(mock_stdout.getvalue(), "Missing filename, key arg, and/or value\n") + self.assertIn("Missing filename, key arg, and/or value\n", mock_stderr.getvalue()) def test_replace(self): filename = "/tmp/so-yaml_test-add.yaml" @@ -360,3 +360,66 @@ class TestRemove(unittest.TestCase): self.assertEqual(soyaml.convertType("false"), False) self.assertEqual(soyaml.convertType("FALSE"), False) self.assertEqual(soyaml.convertType(""), "") + + def test_get_int(self): + with patch('sys.stdout', new=StringIO()) as mock_stdout: + filename = "/tmp/so-yaml_test-get.yaml" + file = open(filename, "w") + file.write("{key1: { child1: 123, child2: { deep1: 45 } }, key2: false, key3: [e,f,g]}") + file.close() + + result = soyaml.get([filename, "key1.child2.deep1"]) + self.assertEqual(result, 0) + self.assertIn("45\n...", mock_stdout.getvalue()) + + def test_get_str(self): + with patch('sys.stdout', new=StringIO()) as mock_stdout: + filename = "/tmp/so-yaml_test-get.yaml" + file = open(filename, "w") + file.write("{key1: { child1: 123, child2: { deep1: \"hello\" } }, key2: false, key3: [e,f,g]}") + file.close() + + result = soyaml.get([filename, "key1.child2.deep1"]) + self.assertEqual(result, 0) + self.assertIn("hello\n...", mock_stdout.getvalue()) + + def test_get_list(self): + with patch('sys.stdout', new=StringIO()) as mock_stdout: + filename = "/tmp/so-yaml_test-get.yaml" + file = open(filename, "w") + file.write("{key1: { child1: 123, child2: { deep1: \"hello\" } }, key2: false, key3: [e,f,g]}") + file.close() + + result = soyaml.get([filename, "key3"]) + self.assertEqual(result, 0) + self.assertIn("- e\n- f\n- g\n", mock_stdout.getvalue()) + + def test_get_dict(self): + with patch('sys.stdout', new=StringIO()) as mock_stdout: + filename = "/tmp/so-yaml_test-get.yaml" + file = open(filename, "w") + file.write("{key1: { child1: 123, child2: { deep1: \"hello\" } }, key2: false, key3: [e,f,g]}") + file.close() + + result = soyaml.get([filename, "key1"]) + self.assertEqual(result, 0) + self.assertIn("child1: 123\nchild2:\n deep1: hello\n", mock_stdout.getvalue()) + + def test_get_missing(self): + with patch('sys.stdout', new=StringIO()) as mock_stdout: + filename = "/tmp/so-yaml_test-get.yaml" + file = open(filename, "w") + file.write("{key1: { child1: 123, child2: { deep1: 45 } }, key2: false, key3: [e,f,g]}") + file.close() + + result = soyaml.get([filename, "key1.child2.deep3"]) + self.assertEqual(result, 2) + self.assertEqual("", mock_stdout.getvalue()) + + def test_get_usage(self): + with patch('sys.exit', new=MagicMock()) as sysmock: + with patch('sys.stderr', new=StringIO()) as mock_stderr: + result = soyaml.get([]) + self.assertEqual(result, 1) + self.assertIn("Missing filename or key arg", mock_stderr.getvalue()) + sysmock.assert_called_once_with(1)