CSEC_PUBLIC/securityonion
1
0
Fork 0
mirror of https://github.com/Security-Onion-Solutions/securityonion.git synced 2025-12-06 09:12:45 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #13150 from Security-Onion-Solutions/jertel/yaml

Browse Source 
add ability to retrieve yaml values via so-yaml.py; improve so-minion id matching
This commit is contained in:
Jason Ertel
2024-06-06 12:09:03 -04:00
committed by GitHub
parent 33a2c5dcd8 5600fed9c4
commit 6d31cd2a41
4 changed files with 149 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
pyci.sh
View File

@@ -15,12 +15,16 @@ TARGET_DIR=${1:-.}
PATH=$PATH:/usr/local/bin
if ! which pytest &> /dev/null || ! which flake8 &> /dev/null ; then
echo "Missing dependencies. Consider running the following command:"
echo " python -m pip install flake8 pytest pytest-cov"
if [ ! -d .venv ]; then
python -m venv .venv
fi
source .venv/bin/activate
if ! pip install flake8 pytest pytest-cov pyyaml; then
echo "Unable to install dependencies."
exit 1
fi
pip install pytest pytest-cov
flake8 "$TARGET_DIR" "--config=${HOME_DIR}/pytest.ini"
python3 -m pytest "--cov-config=${HOME_DIR}/pytest.ini" "--cov=$TARGET_DIR" --doctest-modules --cov-report=term --cov-fail-under=100 "$TARGET_DIR"
python3 -m pytest "--cov-config=${HOME_DIR}/pytest.ini" "--cov=$TARGET_DIR" --doctest-modules --cov-report=term --cov-fail-under=100 "$TARGET_DIR"

4
salt/manager/tools/sbin/so-minion
View File

@@ -112,8 +112,8 @@ function testMinion() {
result=$?
# If this so-minion script is not running on the given minion ID, run so-test remotely on the sensor as well
local_id=$(lookup_grain id)
if [[ ! "$local_id" =~ "${MINION_ID}_" ]]; then
local_id=$(lookup_grain id)
if [[ ! "$local_id" =~ "${MINION_ID}_" && "$local_id" != "${MINION_ID}" ]]; then
salt "$MINION_ID" cmd.run 'so-test'
result=$?
fi

74
salt/manager/tools/sbin/so-yaml.py
View File

@@ -14,19 +14,20 @@ lockFile = "/tmp/so-yaml.lock"
def showUsage(args):
print('Usage: {} <COMMAND> <YAML_FILE> [ARGS...]'.format(sys.argv[0]))
print(' General commands:')
print(' append - Append a list item to a yaml key, if it exists and is a list. Requires KEY and LISTITEM args.')
print(' add - Add a new key and set its value. Fails if key already exists. Requires KEY and VALUE args.')
print(' remove - Removes a yaml key, if it exists. Requires KEY arg.')
print(' replace - Replaces (or adds) a new key and set its value. Requires KEY and VALUE args.')
print(' help - Prints this usage information.')
print('')
print(' Where:')
print(' YAML_FILE - Path to the file that will be modified. Ex: /opt/so/conf/service/conf.yaml')
print(' KEY - YAML key, does not support \' or " characters at this time. Ex: level1.level2')
print(' VALUE - Value to set for a given key')
print(' LISTITEM - Item to append to a given key\'s list value')
print('Usage: {} <COMMAND> <YAML_FILE> [ARGS...]'.format(sys.argv[0]), file=sys.stderr)
print(' General commands:', file=sys.stderr)
print(' append - Append a list item to a yaml key, if it exists and is a list. Requires KEY and LISTITEM args.', file=sys.stderr)
print(' add - Add a new key and set its value. Fails if key already exists. Requires KEY and VALUE args.', file=sys.stderr)
print(' get - Displays (to stdout) the value stored in the given key. Requires KEY arg.', file=sys.stderr)
print(' remove - Removes a yaml key, if it exists. Requires KEY arg.', file=sys.stderr)
print(' replace - Replaces (or adds) a new key and set its value. Requires KEY and VALUE args.', file=sys.stderr)
print(' help - Prints this usage information.', file=sys.stderr)
print('', file=sys.stderr)
print(' Where:', file=sys.stderr)
print(' YAML_FILE - Path to the file that will be modified. Ex: /opt/so/conf/service/conf.yaml', file=sys.stderr)
print(' KEY - YAML key, does not support \' or " characters at this time. Ex: level1.level2', file=sys.stderr)
print(' VALUE - Value to set for a given key', file=sys.stderr)
print(' LISTITEM - Item to append to a given key\'s list value', file=sys.stderr)
sys.exit(1)
@@ -38,7 +39,7 @@ def loadYaml(filename):
def writeYaml(filename, content):
file = open(filename, "w")
return yaml.dump(content, file)
return yaml.safe_dump(content, file)
def appendItem(content, key, listItem):
@@ -49,15 +50,15 @@ def appendItem(content, key, listItem):
try:
content[key].append(listItem)
except AttributeError:
print("The existing value for the given key is not a list. No action was taken on the file.")
print("The existing value for the given key is not a list. No action was taken on the file.", file=sys.stderr)
return 1
except KeyError:
print("The key provided does not exist. No action was taken on the file.")
print("The key provided does not exist. No action was taken on the file.", file=sys.stderr)
return 1
def convertType(value):
if len(value) > 0 and (not value.startswith("0") or len(value) == 1):
if isinstance(value, str) and len(value) > 0 and (not value.startswith("0") or len(value) == 1):
if "." in value:
try:
value = float(value)
@@ -83,7 +84,7 @@ def append(args):
if len(args) != 3:
print('Missing filename, key arg, or list item to append', file=sys.stderr)
showUsage(None)
return
return 1
filename = args[0]
key = args[1]
@@ -112,7 +113,7 @@ def add(args):
if len(args) != 3:
print('Missing filename, key arg, and/or value', file=sys.stderr)
showUsage(None)
return
return 1
filename = args[0]
key = args[1]
@@ -137,7 +138,7 @@ def remove(args):
if len(args) != 2:
print('Missing filename or key arg', file=sys.stderr)
showUsage(None)
return
return 1
filename = args[0]
key = args[1]
@@ -153,7 +154,7 @@ def replace(args):
if len(args) != 3:
print('Missing filename, key arg, and/or value', file=sys.stderr)
showUsage(None)
return
return 1
filename = args[0]
key = args[1]
@@ -167,6 +168,32 @@ def replace(args):
return 0
def getKeyValue(content, key):
pieces = key.split(".", 1)
if len(pieces) > 1:
return getKeyValue(content[pieces[0]], pieces[1])
return content.get(key, None)
def get(args):
if len(args) != 2:
print('Missing filename or key arg', file=sys.stderr)
showUsage(None)
return 1
filename = args[0]
key = args[1]
content = loadYaml(filename)
output = getKeyValue(content, key)
if output is None:
print("Not found", file=sys.stderr)
return 2
print(yaml.safe_dump(output))
return 0
def main():
args = sys.argv[1:]
@@ -178,6 +205,7 @@ def main():
"help": showUsage,
"add": add,
"append": append,
"get": get,
"remove": remove,
"replace": replace,
}
@@ -195,11 +223,11 @@ def main():
break
except Exception:
if lockAttempts == 1:
print("Waiting for lock file to be released from another process...")
print("Waiting for lock file to be released from another process...", file=sys.stderr)
time.sleep(2)
if lockAttempts == maxAttempts:
print("Lock file (" + lockFile + ") could not be created; proceeding without lock.")
print("Lock file (" + lockFile + ") could not be created; proceeding without lock.", file=sys.stderr)
cmd = commands.get(args[0], showUsage)
code = cmd(args[1:])

111
salt/manager/tools/sbin/so-yaml_test.py
View File

@@ -15,40 +15,40 @@ class TestRemove(unittest.TestCase):
def test_main_missing_input(self):
with patch('sys.exit', new=MagicMock()) as sysmock:
with patch('sys.stderr', new=StringIO()) as mock_stdout:
with patch('sys.stderr', new=StringIO()) as mock_stderr:
sys.argv = ["cmd"]
soyaml.main()
sysmock.assert_called_once_with(1)
self.assertIn(mock_stdout.getvalue(), "Usage:")
self.assertIn("Usage:", mock_stderr.getvalue())
def test_main_help_locked(self):
filename = "/tmp/so-yaml.lock"
file = open(filename, "w")
file.write = "fake lock file"
with patch('sys.exit', new=MagicMock()) as sysmock:
with patch('sys.stderr', new=StringIO()) as mock_stdout:
with patch('sys.stderr', new=StringIO()) as mock_stderr:
with patch('time.sleep', new=MagicMock()) as mock_sleep:
sys.argv = ["cmd", "help"]
soyaml.main()
sysmock.assert_called()
mock_sleep.assert_called_with(2)
self.assertIn(mock_stdout.getvalue(), "Usage:")
self.assertIn("Usage:", mock_stderr.getvalue())
def test_main_help(self):
with patch('sys.exit', new=MagicMock()) as sysmock:
with patch('sys.stderr', new=StringIO()) as mock_stdout:
with patch('sys.stderr', new=StringIO()) as mock_stderr:
sys.argv = ["cmd", "help"]
soyaml.main()
sysmock.assert_called()
self.assertIn(mock_stdout.getvalue(), "Usage:")
self.assertIn("Usage:", mock_stderr.getvalue())
def test_remove_missing_arg(self):
with patch('sys.exit', new=MagicMock()) as sysmock:
with patch('sys.stderr', new=StringIO()) as mock_stdout:
with patch('sys.stderr', new=StringIO()) as mock_stderr:
sys.argv = ["cmd", "help"]
soyaml.remove(["file"])
sysmock.assert_called()
self.assertIn(mock_stdout.getvalue(), "Missing filename or key arg\n")
self.assertIn("Missing filename or key arg\n", mock_stderr.getvalue())
def test_remove(self):
filename = "/tmp/so-yaml_test-remove.yaml"
@@ -97,7 +97,7 @@ class TestRemove(unittest.TestCase):
def test_remove_missing_args(self):
with patch('sys.exit', new=MagicMock()) as sysmock:
with patch('sys.stderr', new=StringIO()) as mock_stdout:
with patch('sys.stderr', new=StringIO()) as mock_stderr:
filename = "/tmp/so-yaml_test-remove.yaml"
file = open(filename, "w")
file.write("{key1: { child1: 123, child2: abc }, key2: false}")
@@ -112,15 +112,15 @@ class TestRemove(unittest.TestCase):
expected = "{key1: { child1: 123, child2: abc }, key2: false}"
self.assertEqual(actual, expected)
sysmock.assert_called_once_with(1)
self.assertIn(mock_stdout.getvalue(), "Missing filename or key arg\n")
self.assertIn("Missing filename or key arg\n", mock_stderr.getvalue())
def test_append_missing_arg(self):
with patch('sys.exit', new=MagicMock()) as sysmock:
with patch('sys.stderr', new=StringIO()) as mock_stdout:
with patch('sys.stderr', new=StringIO()) as mock_stderr:
sys.argv = ["cmd", "help"]
soyaml.append(["file", "key"])
sysmock.assert_called()
self.assertIn(mock_stdout.getvalue(), "Missing filename, key arg, or list item to append\n")
self.assertIn("Missing filename, key arg, or list item to append\n", mock_stderr.getvalue())
def test_append(self):
filename = "/tmp/so-yaml_test-remove.yaml"
@@ -173,11 +173,11 @@ class TestRemove(unittest.TestCase):
file.close()
with patch('sys.exit', new=MagicMock()) as sysmock:
with patch('sys.stdout', new=StringIO()) as mock_stdout:
with patch('sys.stderr', new=StringIO()) as mock_stderr:
sys.argv = ["cmd", "append", filename, "key4", "h"]
soyaml.main()
sysmock.assert_called()
self.assertEqual(mock_stdout.getvalue(), "The key provided does not exist. No action was taken on the file.\n")
self.assertEqual("The key provided does not exist. No action was taken on the file.\n", mock_stderr.getvalue())
def test_append_key_noexist_deep(self):
filename = "/tmp/so-yaml_test-append.yaml"
@@ -186,11 +186,11 @@ class TestRemove(unittest.TestCase):
file.close()
with patch('sys.exit', new=MagicMock()) as sysmock:
with patch('sys.stdout', new=StringIO()) as mock_stdout:
with patch('sys.stderr', new=StringIO()) as mock_stderr:
sys.argv = ["cmd", "append", filename, "key1.child2.deep3", "h"]
soyaml.main()
sysmock.assert_called()
self.assertEqual(mock_stdout.getvalue(), "The key provided does not exist. No action was taken on the file.\n")
self.assertEqual("The key provided does not exist. No action was taken on the file.\n", mock_stderr.getvalue())
def test_append_key_nonlist(self):
filename = "/tmp/so-yaml_test-append.yaml"
@@ -199,11 +199,11 @@ class TestRemove(unittest.TestCase):
file.close()
with patch('sys.exit', new=MagicMock()) as sysmock:
with patch('sys.stdout', new=StringIO()) as mock_stdout:
with patch('sys.stderr', new=StringIO()) as mock_stderr:
sys.argv = ["cmd", "append", filename, "key1", "h"]
soyaml.main()
sysmock.assert_called()
self.assertEqual(mock_stdout.getvalue(), "The existing value for the given key is not a list. No action was taken on the file.\n")
self.assertEqual("The existing value for the given key is not a list. No action was taken on the file.\n", mock_stderr.getvalue())
def test_append_key_nonlist_deep(self):
filename = "/tmp/so-yaml_test-append.yaml"
@@ -212,11 +212,11 @@ class TestRemove(unittest.TestCase):
file.close()
with patch('sys.exit', new=MagicMock()) as sysmock:
with patch('sys.stdout', new=StringIO()) as mock_stdout:
with patch('sys.stderr', new=StringIO()) as mock_stderr:
sys.argv = ["cmd", "append", filename, "key1.child2.deep1", "h"]
soyaml.main()
sysmock.assert_called()
self.assertEqual(mock_stdout.getvalue(), "The existing value for the given key is not a list. No action was taken on the file.\n")
self.assertEqual("The existing value for the given key is not a list. No action was taken on the file.\n", mock_stderr.getvalue())
def test_add_key(self):
content = {}
@@ -244,11 +244,11 @@ class TestRemove(unittest.TestCase):
def test_add_missing_arg(self):
with patch('sys.exit', new=MagicMock()) as sysmock:
with patch('sys.stderr', new=StringIO()) as mock_stdout:
with patch('sys.stderr', new=StringIO()) as mock_stderr:
sys.argv = ["cmd", "help"]
soyaml.add(["file", "key"])
sysmock.assert_called()
self.assertIn(mock_stdout.getvalue(), "Missing filename, key arg, and/or value\n")
self.assertIn("Missing filename, key arg, and/or value\n", mock_stderr.getvalue())
def test_add(self):
filename = "/tmp/so-yaml_test-add.yaml"
@@ -296,11 +296,11 @@ class TestRemove(unittest.TestCase):
def test_replace_missing_arg(self):
with patch('sys.exit', new=MagicMock()) as sysmock:
with patch('sys.stderr', new=StringIO()) as mock_stdout:
with patch('sys.stderr', new=StringIO()) as mock_stderr:
sys.argv = ["cmd", "help"]
soyaml.replace(["file", "key"])
sysmock.assert_called()
self.assertIn(mock_stdout.getvalue(), "Missing filename, key arg, and/or value\n")
self.assertIn("Missing filename, key arg, and/or value\n", mock_stderr.getvalue())
def test_replace(self):
filename = "/tmp/so-yaml_test-add.yaml"
@@ -360,3 +360,66 @@ class TestRemove(unittest.TestCase):
self.assertEqual(soyaml.convertType("false"), False)
self.assertEqual(soyaml.convertType("FALSE"), False)
self.assertEqual(soyaml.convertType(""), "")
def test_get_int(self):
with patch('sys.stdout', new=StringIO()) as mock_stdout:
filename = "/tmp/so-yaml_test-get.yaml"
file = open(filename, "w")
file.write("{key1: { child1: 123, child2: { deep1: 45 } }, key2: false, key3: [e,f,g]}")
file.close()
result = soyaml.get([filename, "key1.child2.deep1"])
self.assertEqual(result, 0)
self.assertIn("45\n...", mock_stdout.getvalue())
def test_get_str(self):
with patch('sys.stdout', new=StringIO()) as mock_stdout:
filename = "/tmp/so-yaml_test-get.yaml"
file = open(filename, "w")
file.write("{key1: { child1: 123, child2: { deep1: \"hello\" } }, key2: false, key3: [e,f,g]}")
file.close()
result = soyaml.get([filename, "key1.child2.deep1"])
self.assertEqual(result, 0)
self.assertIn("hello\n...", mock_stdout.getvalue())
def test_get_list(self):
with patch('sys.stdout', new=StringIO()) as mock_stdout:
filename = "/tmp/so-yaml_test-get.yaml"
file = open(filename, "w")
file.write("{key1: { child1: 123, child2: { deep1: \"hello\" } }, key2: false, key3: [e,f,g]}")
file.close()
result = soyaml.get([filename, "key3"])
self.assertEqual(result, 0)
self.assertIn("- e\n- f\n- g\n", mock_stdout.getvalue())
def test_get_dict(self):
with patch('sys.stdout', new=StringIO()) as mock_stdout:
filename = "/tmp/so-yaml_test-get.yaml"
file = open(filename, "w")
file.write("{key1: { child1: 123, child2: { deep1: \"hello\" } }, key2: false, key3: [e,f,g]}")
file.close()
result = soyaml.get([filename, "key1"])
self.assertEqual(result, 0)
self.assertIn("child1: 123\nchild2:\n deep1: hello\n", mock_stdout.getvalue())
def test_get_missing(self):
with patch('sys.stdout', new=StringIO()) as mock_stdout:
filename = "/tmp/so-yaml_test-get.yaml"
file = open(filename, "w")
file.write("{key1: { child1: 123, child2: { deep1: 45 } }, key2: false, key3: [e,f,g]}")
file.close()
result = soyaml.get([filename, "key1.child2.deep3"])
self.assertEqual(result, 2)
self.assertEqual("", mock_stdout.getvalue())
def test_get_usage(self):
with patch('sys.exit', new=MagicMock()) as sysmock:
with patch('sys.stderr', new=StringIO()) as mock_stderr:
result = soyaml.get([])
self.assertEqual(result, 1)
self.assertIn("Missing filename or key arg", mock_stderr.getvalue())
sysmock.assert_called_once_with(1)