Add test for raw dict output in so-yaml get to reach 100% coverage

Covers the dict/list branch in raw mode (line 358) that was missing
test coverage.

.github/DISCUSSION_TEMPLATE/2-4.yml

@@ -33,7 +33,7 @@ body:
         - 2.4.200
         - 2.4.201
         - 2.4.210
-        - 3.0.0
+        - 2.4.211
         - Other (please provide detail below)
     validations:
       required: true

.github/DISCUSSION_TEMPLATE/3-0.yml

@@ -0,0 +1,177 @@
+body:
+  - type: markdown
+    attributes:
+      value: |
+        If your organization needs more immediate, enterprise grade professional support, with one-on-one virtual meetings and screensharing, contact us via our website: https://securityonion.com/support
+  - type: dropdown
+    attributes:
+      label: Version
+      description: Which version of Security Onion are you asking about?
+      options:
+        -
+        - 3.0.0
+        - Other (please provide detail below)
+    validations:
+      required: true
+  - type: dropdown
+    attributes:
+      label: Installation Method
+      description: How did you install Security Onion?
+      options:
+        -
+        - Security Onion ISO image
+        - Cloud image (Amazon, Azure, Google)
+        - Network installation on Oracle 9 (unsupported)
+        - Other (please provide detail below)
+    validations:
+      required: true
+  - type: dropdown
+    attributes:
+      label: Description
+      description: >
+        Is this discussion about installation, configuration, upgrading, or other?
+      options:
+        -
+        - installation
+        - configuration
+        - upgrading
+        - other (please provide detail below)
+    validations:
+      required: true
+  - type: dropdown
+    attributes:
+      label: Installation Type
+      description: >
+        When you installed, did you choose Import, Eval, Standalone, Distributed, or something else?
+      options:
+        -
+        - Import
+        - Eval
+        - Standalone
+        - Distributed
+        - other (please provide detail below)
+    validations:
+      required: true
+  - type: dropdown
+    attributes:
+      label: Location
+      description: >
+        Is this deployment in the cloud, on-prem with Internet access, or airgap?
+      options:
+        -
+        - cloud
+        - on-prem with Internet access
+        - airgap
+        - other (please provide detail below)
+    validations:
+      required: true
+  - type: dropdown
+    attributes:
+      label: Hardware Specs
+      description: >
+        Does your hardware meet or exceed the minimum requirements for your installation type as shown at https://securityonion.net/docs/hardware?
+      options:
+        -
+        - Meets minimum requirements
+        - Exceeds minimum requirements
+        - Does not meet minimum requirements
+        - other (please provide detail below)
+    validations:
+      required: true
+  - type: input
+    attributes:
+      label: CPU
+      description: How many CPU cores do you have?
+    validations:
+      required: true
+  - type: input
+    attributes:
+      label: RAM
+      description: How much RAM do you have?
+    validations:
+      required: true
+  - type: input
+    attributes:
+      label: Storage for /
+      description: How much storage do you have for the / partition?
+    validations:
+      required: true
+  - type: input
+    attributes:
+      label: Storage for /nsm
+      description: How much storage do you have for the /nsm partition?
+    validations:
+      required: true
+  - type: dropdown
+    attributes:
+      label: Network Traffic Collection
+      description: >
+        Are you collecting network traffic from a tap or span port?
+      options:
+        -
+        - tap
+        - span port
+        - other (please provide detail below)
+    validations:
+      required: true
+  - type: dropdown
+    attributes:
+      label: Network Traffic Speeds
+      description: >
+        How much network traffic are you monitoring?
+      options:
+        -
+        - Less than 1Gbps
+        - 1Gbps to 10Gbps
+        - more than 10Gbps
+    validations:
+      required: true
+  - type: dropdown
+    attributes:
+      label: Status
+      description: >
+        Does SOC Grid show all services on all nodes as running OK?
+      options:
+        -
+        - Yes, all services on all nodes are running OK
+        - No, one or more services are failed (please provide detail below)
+    validations:
+      required: true
+  - type: dropdown
+    attributes:
+      label: Salt Status
+      description: >
+        Do you get any failures when you run "sudo salt-call state.highstate"?
+      options:
+        -
+        - Yes, there are salt failures (please provide detail below)
+        - No, there are no failures
+    validations:
+      required: true
+  - type: dropdown
+    attributes:
+      label: Logs
+      description: >
+        Are there any additional clues in /opt/so/log/?
+      options:
+        -
+        - Yes, there are additional clues in /opt/so/log/ (please provide detail below)
+        - No, there are no additional clues
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: Detail
+      description: Please read our discussion guidelines at https://github.com/Security-Onion-Solutions/securityonion/discussions/1720 and then provide detailed information to help us help you.
+      placeholder: |-
+        STOP! Before typing, please read our discussion guidelines at https://github.com/Security-Onion-Solutions/securityonion/discussions/1720 in their entirety!
+
+        If your organization needs more immediate, enterprise grade professional support, with one-on-one virtual meetings and screensharing, contact us via our website: https://securityonion.com/support
+    validations:
+      required: true
+  - type: checkboxes
+    attributes:
+      label: Guidelines
+      options:
+        - label: I have read the discussion guidelines at https://github.com/Security-Onion-Solutions/securityonion/discussions/1720 and assert that I have followed the guidelines.
+          required: true

salt/manager/tools/sbin/so-minion

@@ -462,19 +462,14 @@ function add_sensor_to_minion() {
         echo "      lb_procs: '$CORECOUNT'"
         echo "suricata:"
         echo "  enabled: True "
+        echo "  pcap:"
+        echo "    enabled: True"
         if [[ $is_pcaplimit ]]; then
-            echo "  pcap:"
             echo "    maxsize: $MAX_PCAP_SPACE"
         fi
         echo "  config:"
         echo "    af-packet:"
         echo "      threads: '$CORECOUNT'"
-        echo "pcap:"
-        echo "  enabled: True"
-        if [[ $is_pcaplimit ]]; then
-            echo "  config:"
-            echo "    diskfreepercentage: $DFREEPERCENT"
-        fi
         echo "  "
     } >> $PILLARFILE
     if [ $? -ne 0 ]; then

salt/manager/tools/sbin/so-yaml.py

@@ -22,7 +22,7 @@ def showUsage(args):
     print('    removelistitem   - Remove a list item from a yaml key, if it exists and is a list. Requires KEY and LISTITEM args.', file=sys.stderr)
     print('    replacelistobject  - Replace a list object based on a condition. Requires KEY, CONDITION_FIELD, CONDITION_VALUE, and JSON_OBJECT args.', file=sys.stderr)
     print('    add              - Add a new key and set its value. Fails if key already exists. Requires KEY and VALUE args.', file=sys.stderr)
-    print('    get              - Displays (to stdout) the value stored in the given key. Requires KEY arg.', file=sys.stderr)
+    print('    get [-r]         - Displays (to stdout) the value stored in the given key. Requires KEY arg. Use -r for raw output without YAML formatting.', file=sys.stderr)
     print('    remove           - Removes a yaml key, if it exists. Requires KEY arg.', file=sys.stderr)
     print('    replace          - Replaces (or adds) a new key and set its value. Requires KEY and VALUE args.', file=sys.stderr)
     print('    help             - Prints this usage information.', file=sys.stderr)
@@ -256,7 +256,7 @@ def replacelistobject(args):
 def addKey(content, key, value):
     pieces = key.split(".", 1)
     if len(pieces) > 1:
-        if not pieces[0] in content:
+        if pieces[0] not in content or content[pieces[0]] is None:
             content[pieces[0]] = {}
         addKey(content[pieces[0]], pieces[1], value)
     elif key in content:
@@ -332,6 +332,11 @@ def getKeyValue(content, key):
 
 
 def get(args):
+    raw = False
+    if len(args) > 0 and args[0] == '-r':
+        raw = True
+        args = args[1:]
+
     if len(args) != 2:
         print('Missing filename or key arg', file=sys.stderr)
         showUsage(None)
@@ -346,7 +351,15 @@ def get(args):
         print(f"Key '{key}' not found by so-yaml.py", file=sys.stderr)
         return 2
 
-    print(yaml.safe_dump(output))
+    if raw:
+        if isinstance(output, bool):
+            print(str(output).lower())
+        elif isinstance(output, (dict, list)):
+            print(yaml.safe_dump(output).strip())
+        else:
+            print(output)
+    else:
+        print(yaml.safe_dump(output))
     return 0
 
 

salt/manager/tools/sbin/so-yaml_test.py

@@ -395,6 +395,17 @@ class TestRemove(unittest.TestCase):
             self.assertEqual(result, 0)
             self.assertIn("45\n...", mock_stdout.getvalue())
 
+    def test_get_int_raw(self):
+        with patch('sys.stdout', new=StringIO()) as mock_stdout:
+            filename = "/tmp/so-yaml_test-get.yaml"
+            file = open(filename, "w")
+            file.write("{key1: { child1: 123, child2: { deep1: 45 } }, key2: false, key3: [e,f,g]}")
+            file.close()
+
+            result = soyaml.get(["-r", filename, "key1.child2.deep1"])
+            self.assertEqual(result, 0)
+            self.assertEqual("45\n", mock_stdout.getvalue())
+
     def test_get_str(self):
         with patch('sys.stdout', new=StringIO()) as mock_stdout:
             filename = "/tmp/so-yaml_test-get.yaml"
@@ -406,6 +417,51 @@ class TestRemove(unittest.TestCase):
             self.assertEqual(result, 0)
             self.assertIn("hello\n...", mock_stdout.getvalue())
 
+    def test_get_str_raw(self):
+        with patch('sys.stdout', new=StringIO()) as mock_stdout:
+            filename = "/tmp/so-yaml_test-get.yaml"
+            file = open(filename, "w")
+            file.write("{key1: { child1: 123, child2: { deep1: \"hello\" } }, key2: false, key3: [e,f,g]}")
+            file.close()
+
+            result = soyaml.get(["-r", filename, "key1.child2.deep1"])
+            self.assertEqual(result, 0)
+            self.assertEqual("hello\n", mock_stdout.getvalue())
+
+    def test_get_bool(self):
+        with patch('sys.stdout', new=StringIO()) as mock_stdout:
+            filename = "/tmp/so-yaml_test-get.yaml"
+            file = open(filename, "w")
+            file.write("{key1: { child1: 123, child2: { deep1: 45 } }, key2: false, key3: [e,f,g]}")
+            file.close()
+
+            result = soyaml.get([filename, "key2"])
+            self.assertEqual(result, 0)
+            self.assertIn("false\n...", mock_stdout.getvalue())
+
+    def test_get_bool_raw(self):
+        with patch('sys.stdout', new=StringIO()) as mock_stdout:
+            filename = "/tmp/so-yaml_test-get.yaml"
+            file = open(filename, "w")
+            file.write("{key1: { child1: 123, child2: { deep1: 45 } }, key2: false, key3: [e,f,g]}")
+            file.close()
+
+            result = soyaml.get(["-r", filename, "key2"])
+            self.assertEqual(result, 0)
+            self.assertEqual("false\n", mock_stdout.getvalue())
+
+    def test_get_dict_raw(self):
+        with patch('sys.stdout', new=StringIO()) as mock_stdout:
+            filename = "/tmp/so-yaml_test-get.yaml"
+            file = open(filename, "w")
+            file.write("{key1: { child1: 123, child2: { deep1: 45 } }, key2: false, key3: [e,f,g]}")
+            file.close()
+
+            result = soyaml.get(["-r", filename, "key1"])
+            self.assertEqual(result, 0)
+            self.assertIn("child1: 123", mock_stdout.getvalue())
+            self.assertNotIn("...", mock_stdout.getvalue())
+
     def test_get_list(self):
         with patch('sys.stdout', new=StringIO()) as mock_stdout:
             filename = "/tmp/so-yaml_test-get.yaml"

salt/manager/tools/sbin/soup

@@ -362,7 +362,7 @@ preupgrade_changes() {
     # This function is to add any new pillar items if needed.
     echo "Checking to see if changes are needed."
 
-    [[ "$INSTALLEDVERSION" == 2.4.210 ]] && up_to_3.0.0   
+    [[ "$INSTALLEDVERSION" =~ ^2\.4\.21[0-9]+$ ]] && up_to_3.0.0   
     true
 }
 
@@ -370,12 +370,12 @@ postupgrade_changes() {
     # This function is to add any new pillar items if needed.
     echo "Running post upgrade processes."
 
-    [[ "$POSTVERSION" == 2.4.210 ]] && post_to_3.0.0
+    [[ "$POSTVERSION" =~ ^2\.4\.21[0-9]+$ ]] && post_to_3.0.0
     true
 }
 
 check_minimum_version() {
-  if [[ "$INSTALLEDVERSION" != "2.4.210" ]] && [[ "$INSTALLEDVERSION" != "2.4.211" ]] && [[ ! "$INSTALLEDVERSION" =~ ^3\. ]]; then
+  if [[ ! "$INSTALLEDVERSION" =~ ^(2\.4\.21[0-9]+|3\.) ]]; then
     echo "You must be on at least Security Onion 2.4.210 to upgrade. Currently installed version: $INSTALLEDVERSION"
     exit 1
   fi
@@ -385,10 +385,23 @@ check_minimum_version() {
 
 up_to_3.0.0() {
   determine_elastic_agent_upgrade
+  migrate_pcap_to_suricata
 
   INSTALLEDVERSION=3.0.0
 }
 
+migrate_pcap_to_suricata() {
+  local MINIONDIR=/opt/so/saltstack/local/pillar/minions
+  local PCAPFILE=/opt/so/saltstack/local/pillar/pcap/soc_pcap.sls
+
+  for pillar_file in "$PCAPFILE" "$MINIONDIR"/*.sls; do
+    [[ -f "$pillar_file" ]] || continue
+    pcap_enabled=$(so-yaml.py get -r "$pillar_file" pcap.enabled 2>/dev/null) || continue
+    so-yaml.py add "$pillar_file" suricata.pcap.enabled "$pcap_enabled"
+    so-yaml.py remove "$pillar_file" pcap
+  done
+}
+
 post_to_3.0.0() {
   echo "Nothing to apply"
   POSTVERSION=3.0.0