Merge pull request #11742 from Security-Onion-Solutions/jertel/yaml

add yaml helper script; refactor python testing

.github/workflows/pythontest.yml

@@ -4,9 +4,11 @@ on:
   push:
     paths: 
       - "salt/sensoroni/files/analyzers/**"
+      - "salt/manager/tools/sbin"
   pull_request:
     paths:
       - "salt/sensoroni/files/analyzers/**"
+      - "salt/manager/tools/sbin"
 
 jobs:
   build:
@@ -16,7 +18,7 @@ jobs:
       fail-fast: false
       matrix:
         python-version: ["3.10"]
-        python-code-path: ["salt/sensoroni/files/analyzers"]
+        python-code-path: ["salt/sensoroni/files/analyzers", "salt/manager/tools/sbin"]
 
     steps:
     - uses: actions/checkout@v3
@@ -34,4 +36,4 @@ jobs:
         flake8 ${{ matrix.python-code-path }} --show-source --max-complexity=12  --doctests --max-line-length=200 --statistics
     - name: Test with pytest
       run: |
-        pytest ${{ matrix.python-code-path }} --cov=${{ matrix.python-code-path }} --doctest-modules --cov-report=term --cov-fail-under=100 --cov-config=${{ matrix.python-code-path }}/pytest.ini
+        pytest ${{ matrix.python-code-path }} --cov=${{ matrix.python-code-path }} --doctest-modules --cov-report=term --cov-fail-under=100 --cov-config=pytest.ini

pyci.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+if [[ $# -ne 1 ]]; then
+    echo "Usage: $0 <python_script_dir>"
+    echo "Runs tests on all *_test.py files in the given directory."
+    exit 1
+fi
+
+HOME_DIR=$(dirname "$0")
+TARGET_DIR=${1:-.}
+
+PATH=$PATH:/usr/local/bin
+
+if ! which pytest &> /dev/null || ! which flake8 &> /dev/null ; then
+    echo "Missing dependencies. Consider running the following command:"
+    echo "  python -m pip install flake8 pytest pytest-cov"
+    exit 1
+fi
+
+pip install pytest pytest-cov
+flake8 "$TARGET_DIR" "--config=${HOME_DIR}/pytest.ini"
+python3 -m pytest "--cov-config=${HOME_DIR}/pytest.ini" "--cov=$TARGET_DIR" --doctest-modules --cov-report=term --cov-fail-under=100  "$TARGET_DIR" 

salt/manager/init.sls

@@ -61,6 +61,8 @@ manager_sbin:
     - user: 939
     - group: 939
     - file_mode: 755
+    - exclude_pat: 
+      - "*_test.py"
 
 yara_update_scripts:
   file.recurse:

salt/manager/tools/sbin/so-firewall

@@ -1,19 +1,9 @@
 #!/usr/bin/env python3
 
-# Copyright 2014-2023 Security Onion Solutions, LLC
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
 
 import os
 import subprocess

salt/manager/tools/sbin/so-yaml.py

@@ -0,0 +1,95 @@
+#!/usr/bin/env python3
+
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+import os
+import sys
+import time
+import yaml
+
+lockFile = "/tmp/so-yaml.lock"
+
+
+def showUsage(args):
+    print('Usage: {} <COMMAND> <YAML_FILE> [ARGS...]'.format(sys.argv[0]))
+    print('  General commands:')
+    print('    remove         - Removes a yaml top-level key, if it exists. Requires KEY arg.')
+    print('    help           - Prints this usage information.')
+    print('')
+    print('  Where:')
+    print('   YAML_FILE       - Path to the file that will be modified. Ex: /opt/so/conf/service/conf.yaml')
+    print('   KEY             - Top level key only, does not support dot-notations for nested keys at this time. Ex: level1')
+    sys.exit(1)
+
+
+def loadYaml(filename):
+    file = open(filename, "r")
+    content = file.read()
+    return yaml.safe_load(content)
+
+
+def writeYaml(filename, content):
+    file = open(filename, "w")
+    return yaml.dump(content, file)
+
+
+def remove(args):
+    if len(args) != 2:
+        print('Missing filename or key arg', file=sys.stderr)
+        showUsage(None)
+        return
+
+    filename = args[0]
+    content = loadYaml(filename)
+
+    content.pop(args[1], None)
+
+    writeYaml(filename, content)
+    return 0
+
+
+def main():
+    args = sys.argv[1:]
+
+    if len(args) < 1:
+        showUsage(None)
+        return
+
+    commands = {
+        "help": showUsage,
+        "remove": remove,
+    }
+
+    code = 1
+
+    try:
+        lockAttempts = 0
+        maxAttempts = 30
+        while lockAttempts < maxAttempts:
+            lockAttempts = lockAttempts + 1
+            try:
+                f = open(lockFile, "x")
+                f.close()
+                break
+            except Exception:
+                if lockAttempts == 1:
+                    print("Waiting for lock file to be released from another process...")
+                time.sleep(2)
+
+        if lockAttempts == maxAttempts:
+            print("Lock file (" + lockFile + ") could not be created; proceeding without lock.")
+
+        cmd = commands.get(args[0], showUsage)
+        code = cmd(args[1:])
+    finally:
+        if os.path.exists(lockFile):
+            os.remove(lockFile)
+
+    sys.exit(code)
+
+
+if __name__ == "__main__":
+    main()

salt/manager/tools/sbin/so-yaml_test.py

@@ -0,0 +1,77 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+from io import StringIO
+import sys
+from unittest.mock import patch, MagicMock
+import unittest
+import importlib
+soyaml = importlib.import_module("so-yaml")
+
+
+class TestRemove(unittest.TestCase):
+
+    def test_main_missing_input(self):
+        with patch('sys.exit', new=MagicMock()) as sysmock:
+            with patch('sys.stderr', new=StringIO()) as mock_stdout:
+                sys.argv = ["cmd"]
+                soyaml.main()
+                sysmock.assert_called_once_with(1)
+                self.assertIn(mock_stdout.getvalue(), "Usage:")
+
+    def test_main_help_locked(self):
+        filename = "/tmp/so-yaml.lock"
+        file = open(filename, "w")
+        file.write = "fake lock file"
+        with patch('sys.exit', new=MagicMock()) as sysmock:
+            with patch('sys.stderr', new=StringIO()) as mock_stdout:
+                with patch('time.sleep', new=MagicMock()) as mock_sleep:
+                    sys.argv = ["cmd", "help"]
+                    soyaml.main()
+                    sysmock.assert_called()
+                    mock_sleep.assert_called_with(2)
+                    self.assertIn(mock_stdout.getvalue(), "Usage:")
+
+    def test_main_help(self):
+        with patch('sys.exit', new=MagicMock()) as sysmock:
+            with patch('sys.stderr', new=StringIO()) as mock_stdout:
+                sys.argv = ["cmd", "help"]
+                soyaml.main()
+                sysmock.assert_called()
+                self.assertIn(mock_stdout.getvalue(), "Usage:")
+
+    def test_remove(self):
+        filename = "/tmp/so-yaml_test-remove.yaml"
+        file = open(filename, "w")
+        file.write("{key1: { child1: 123, child2: abc }, key2: false}")
+        file.close()
+
+        soyaml.remove([filename, "key1"])
+
+        file = open(filename, "r")
+        actual = file.read()
+        file.close()
+
+        expected = "key2: false\n"
+        self.assertEqual(actual, expected)
+
+    def test_remove_missing_args(self):
+        with patch('sys.exit', new=MagicMock()) as sysmock:
+            with patch('sys.stderr', new=StringIO()) as mock_stdout:
+                filename = "/tmp/so-yaml_test-remove.yaml"
+                file = open(filename, "w")
+                file.write("{key1: { child1: 123, child2: abc }, key2: false}")
+                file.close()
+
+                soyaml.remove([filename])
+
+                file = open(filename, "r")
+                actual = file.read()
+                file.close()
+
+                expected = "{key1: { child1: 123, child2: abc }, key2: false}"
+                self.assertEqual(actual, expected)
+                sysmock.assert_called_once_with(1)
+                self.assertIn(mock_stdout.getvalue(), "Missing filename or key arg\n")

salt/manager/tools/sbin_jinja/so-elastic-fleet-reset

@@ -29,7 +29,7 @@ status "Stopping Fleet Container..."
 so-elastic-fleet-stop --force
 
 status "Deleting Fleet Data from Pillars..."
-sed -i -z "s/elasticfleet:.*grid_enrollment_heavy.*'//" /opt/so/saltstack/local/pillar/minions/{{ GLOBALS.minion_id }}.sls
+so-yaml.py remove /opt/so/saltstack/local/pillar/minions/{{ GLOBALS.minion_id }}.sls elasticfleet
 sed -i "/fleet_grid_enrollment_token_general.*/d" /opt/so/saltstack/local/pillar/global/soc_global.sls
 sed -i "/fleet_grid_enrollment_token_heavy.*/d" /opt/so/saltstack/local/pillar/global/soc_global.sls
 

salt/sensoroni/files/analyzers/build.sh

@@ -3,23 +3,6 @@
 COMMAND=$1
 SENSORONI_CONTAINER=${SENSORONI_CONTAINER:-so-sensoroni}
 
-function ci() {
-    HOME_DIR=$(dirname "$0")
-    TARGET_DIR=${1:-.}
-
-    PATH=$PATH:/usr/local/bin
-
-    if ! which pytest &> /dev/null || ! which flake8 &> /dev/null ; then
-        echo "Missing dependencies. Consider running the following command:"
-        echo "  python -m pip install flake8 pytest pytest-cov"
-        exit 1
-    fi
-
-    pip install pytest pytest-cov
-    flake8 "$TARGET_DIR" "--config=${HOME_DIR}/pytest.ini"
-    python3 -m pytest "--cov-config=${HOME_DIR}/pytest.ini" "--cov=$TARGET_DIR" --doctest-modules --cov-report=term --cov-fail-under=100  "$TARGET_DIR" 
-}
-
 function download() {
     ANALYZERS=$1
     if [[ $ANALYZERS = "all" ]]; then
@@ -36,5 +19,5 @@ function download() {
 if [[ "$COMMAND" == "download" ]]; then
   download "$2"
 else
-  ci
+  ../../../../pyci.sh $@
 fi