Merge pull request #16280 from Security-Onion-Solutions/cogburn/unified-automations

Unified Automations

salt/elasticsearch/defaults.yaml

@@ -3309,7 +3309,6 @@ elasticsearch:
         composed_of:
         - event-mappings
         - logs-system.security@package
-        - so-fleet_system.security_caseless-1
         - logs-system.security@custom
         - so-fleet_integrations.ip_mappings-1
         - so-fleet_globals-1
@@ -4176,7 +4175,6 @@ elasticsearch:
       index_template:
         composed_of:
         - logs-windows.forwarded@package
-        - so-fleet_process_caseless-1
         - logs-windows.forwarded@custom
         - so-fleet_integrations.ip_mappings-1
         - so-fleet_globals-1
@@ -4226,7 +4224,6 @@ elasticsearch:
       index_template:
         composed_of:
         - logs-windows.powershell@package
-        - so-fleet_process_caseless-1
         - logs-windows.powershell@custom
         - so-fleet_integrations.ip_mappings-1
         - so-fleet_globals-1
@@ -4276,7 +4273,6 @@ elasticsearch:
       index_template:
         composed_of:
         - logs-windows.powershell_operational@package
-        - so-fleet_process_caseless-1
         - logs-windows.powershell_operational@custom
         - so-fleet_integrations.ip_mappings-1
         - so-fleet_globals-1
@@ -4326,7 +4322,6 @@ elasticsearch:
       index_template:
         composed_of:
         - logs-windows.sysmon_operational@package
-        - so-fleet_process_caseless-1
         - logs-windows.sysmon_operational@custom
         - so-fleet_integrations.ip_mappings-1
         - so-fleet_globals-1

salt/elasticsearch/templates/component/elastic-agent/so-fleet_process_caseless-1.json

@@ -1,123 +0,0 @@
-{
-  "_meta": {
-    "managed_by": "security_onion",
-    "managed": true,
-    "description": "Adds .caseless for Lucene queries. Restates each field's package type and .text."
-  },
-  "template": {
-    "mappings": {
-      "properties": {
-        "process": {
-          "properties": {
-            "executable": {
-              "type": "keyword",
-              "ignore_above": 1024,
-              "fields": {
-                "caseless": {
-                  "type": "keyword",
-                  "ignore_above": 1024,
-                  "normalizer": "lowercase"
-                },
-                "text": {
-                  "type": "match_only_text"
-                }
-              }
-            },
-            "name": {
-              "type": "keyword",
-              "ignore_above": 1024,
-              "fields": {
-                "caseless": {
-                  "type": "keyword",
-                  "ignore_above": 1024,
-                  "normalizer": "lowercase"
-                },
-                "text": {
-                  "type": "match_only_text"
-                }
-              }
-            },
-            "command_line": {
-              "type": "wildcard",
-              "ignore_above": 1024,
-              "fields": {
-                "caseless": {
-                  "type": "keyword",
-                  "ignore_above": 1024,
-                  "normalizer": "lowercase"
-                },
-                "text": {
-                  "type": "match_only_text"
-                }
-              }
-            },
-            "parent": {
-              "properties": {
-                "executable": {
-                  "type": "keyword",
-                  "ignore_above": 1024,
-                  "fields": {
-                    "caseless": {
-                      "type": "keyword",
-                      "ignore_above": 1024,
-                      "normalizer": "lowercase"
-                    },
-                    "text": {
-                      "type": "match_only_text"
-                    }
-                  }
-                },
-                "name": {
-                  "type": "keyword",
-                  "ignore_above": 1024,
-                  "fields": {
-                    "caseless": {
-                      "type": "keyword",
-                      "ignore_above": 1024,
-                      "normalizer": "lowercase"
-                    },
-                    "text": {
-                      "type": "match_only_text"
-                    }
-                  }
-                },
-                "command_line": {
-                  "type": "wildcard",
-                  "ignore_above": 1024,
-                  "fields": {
-                    "caseless": {
-                      "type": "keyword",
-                      "ignore_above": 1024,
-                      "normalizer": "lowercase"
-                    },
-                    "text": {
-                      "type": "match_only_text"
-                    }
-                  }
-                }
-              }
-            }
-          }
-        },
-        "file": {
-          "properties": {
-            "path": {
-              "type": "keyword",
-              "ignore_above": 1024,
-              "fields": {
-                "caseless": {
-                  "type": "keyword",
-                  "ignore_above": 1024,
-                  "normalizer": "lowercase"
-                },
-                "text": {
-                  "type": "match_only_text"
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  }
-}

salt/elasticsearch/templates/component/elastic-agent/so-fleet_system.security_caseless-1.json

@@ -1,80 +0,0 @@
-{
-  "_meta": {
-    "managed_by": "security_onion",
-    "managed": true,
-    "description": "Adds .caseless for Lucene queries. Keeps each field's existing keyword type."
-  },
-  "template": {
-    "mappings": {
-      "properties": {
-        "process": {
-          "properties": {
-            "command_line": {
-              "type": "keyword",
-              "ignore_above": 1024,
-              "fields": {
-                "caseless": {
-                  "type": "keyword",
-                  "ignore_above": 1024,
-                  "normalizer": "lowercase"
-                }
-              }
-            },
-            "parent": {
-              "properties": {
-                "executable": {
-                  "type": "keyword",
-                  "ignore_above": 1024,
-                  "fields": {
-                    "caseless": {
-                      "type": "keyword",
-                      "ignore_above": 1024,
-                      "normalizer": "lowercase"
-                    }
-                  }
-                },
-                "name": {
-                  "type": "keyword",
-                  "ignore_above": 1024,
-                  "fields": {
-                    "caseless": {
-                      "type": "keyword",
-                      "ignore_above": 1024,
-                      "normalizer": "lowercase"
-                    }
-                  }
-                },
-                "command_line": {
-                  "type": "keyword",
-                  "ignore_above": 1024,
-                  "fields": {
-                    "caseless": {
-                      "type": "keyword",
-                      "ignore_above": 1024,
-                      "normalizer": "lowercase"
-                    }
-                  }
-                }
-              }
-            }
-          }
-        },
-        "file": {
-          "properties": {
-            "path": {
-              "type": "keyword",
-              "ignore_above": 1024,
-              "fields": {
-                "caseless": {
-                  "type": "keyword",
-                  "ignore_above": 1024,
-                  "normalizer": "lowercase"
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  }
-}

salt/manager/tools/sbin/so-yaml.py

@@ -42,7 +42,8 @@ def loadYaml(filename):
     try:
         with open(filename, "r") as file:
             content = file.read()
-            return yaml.safe_load(content)
+            loaded = yaml.safe_load(content)
+            return loaded if loaded is not None else {}
     except FileNotFoundError:
         print(f"File not found: {filename}", file=sys.stderr)
         sys.exit(1)

salt/manager/tools/sbin/so-yaml_test.py

@@ -95,6 +95,20 @@ class TestRemove(unittest.TestCase):
         expected = "key1:\n  child1: 123\n  child2:\n    deep2: ab\nkey2: false\n"
         self.assertEqual(actual, expected)
 
+    def test_remove_empty_file(self):
+        filename = "/tmp/so-yaml_test-remove-empty.yaml"
+        file = open(filename, "w")
+        file.close()
+
+        code = soyaml.remove([filename, "key1"])
+        self.assertEqual(code, 0)
+
+        file = open(filename, "r")
+        actual = file.read()
+        file.close()
+
+        self.assertEqual(actual, "{}\n")
+
     def test_remove_missing_args(self):
         with patch('sys.exit', new=MagicMock()) as sysmock:
             with patch('sys.stderr', new=StringIO()) as mock_stderr:
@@ -294,6 +308,36 @@ class TestRemove(unittest.TestCase):
         expected = "key1:\n  child1: 123\n  child2:\n    deep1: 45\n    deep2: d\nkey2: false\nkey3:\n- e\n- f\n- g\n"
         self.assertEqual(actual, expected)
 
+    def test_add_empty_file(self):
+        filename = "/tmp/so-yaml_test-add-empty.yaml"
+        file = open(filename, "w")
+        file.close()
+
+        code = soyaml.add([filename, "telegraf.output", "BOTH"])
+        self.assertEqual(code, 0)
+
+        file = open(filename, "r")
+        actual = file.read()
+        file.close()
+
+        expected = "telegraf:\n  output: BOTH\n"
+        self.assertEqual(actual, expected)
+
+    def test_add_empty_file_simple(self):
+        filename = "/tmp/so-yaml_test-add-empty-simple.yaml"
+        file = open(filename, "w")
+        file.close()
+
+        code = soyaml.add([filename, "telegraf", "BOTH"])
+        self.assertEqual(code, 0)
+
+        file = open(filename, "r")
+        actual = file.read()
+        file.close()
+
+        expected = "telegraf: BOTH\n"
+        self.assertEqual(actual, expected)
+
     def test_replace_missing_arg(self):
         with patch('sys.exit', new=MagicMock()) as sysmock:
             with patch('sys.stderr', new=StringIO()) as mock_stderr:
@@ -346,6 +390,21 @@ class TestRemove(unittest.TestCase):
         expected = "key1:\n  child1: 123\n  child2:\n    deep1: 46\nkey2: false\nkey3:\n- e\n- f\n- g\n"
         self.assertEqual(actual, expected)
 
+    def test_replace_empty_file(self):
+        filename = "/tmp/so-yaml_test-replace-empty.yaml"
+        file = open(filename, "w")
+        file.close()
+
+        code = soyaml.replace([filename, "telegraf.output", "BOTH"])
+        self.assertEqual(code, 0)
+
+        file = open(filename, "r")
+        actual = file.read()
+        file.close()
+
+        expected = "telegraf:\n  output: BOTH\n"
+        self.assertEqual(actual, expected)
+
     def test_convert(self):
         self.assertEqual(soyaml.convertType("foo"), "foo")
         self.assertEqual(soyaml.convertType("foo.bar"), "foo.bar")
@@ -506,6 +565,18 @@ class TestRemove(unittest.TestCase):
             self.assertEqual(result, 2)
             self.assertEqual("", mock_stdout.getvalue())
 
+    def test_get_empty_file(self):
+        with patch('sys.stdout', new=StringIO()) as mock_stdout:
+            with patch('sys.stderr', new=StringIO()) as mock_stderr:
+                filename = "/tmp/so-yaml_test-get-empty.yaml"
+                file = open(filename, "w")
+                file.close()
+
+                result = soyaml.get([filename, "telegraf.output"])
+                self.assertEqual(result, 2)
+                self.assertEqual("", mock_stdout.getvalue())
+                self.assertIn("Key 'telegraf.output' not found by so-yaml.py", mock_stderr.getvalue())
+
     def test_get_usage(self):
         with patch('sys.exit', new=MagicMock()) as sysmock:
             with patch('sys.stderr', new=StringIO()) as mock_stderr:
@@ -991,3 +1062,29 @@ class TestLoadYaml(unittest.TestCase):
                     soyaml.loadYaml("/tmp/so-yaml_test-unreadable.yaml")
                     sysmock.assert_called_with(1)
                     self.assertIn("Error reading file", mock_stderr.getvalue())
+
+    def test_load_yaml_empty_file(self):
+        filename = "/tmp/so-yaml_test-load-empty.yaml"
+        file = open(filename, "w")
+        file.close()
+
+        result = soyaml.loadYaml(filename)
+        self.assertEqual(result, {})
+
+    def test_load_yaml_whitespace_only(self):
+        filename = "/tmp/so-yaml_test-load-whitespace.yaml"
+        file = open(filename, "w")
+        file.write("   \n\n  \n")
+        file.close()
+
+        result = soyaml.loadYaml(filename)
+        self.assertEqual(result, {})
+
+    def test_load_yaml_comments_only(self):
+        filename = "/tmp/so-yaml_test-load-comments.yaml"
+        file = open(filename, "w")
+        file.write("# Just a comment\n# Another comment\n")
+        file.close()
+
+        result = soyaml.loadYaml(filename)
+        self.assertEqual(result, {})

salt/soc/files/soc/sigma_playbook_pipeline.yaml

@@ -2,6 +2,7 @@ name: Security Onion - Playbook Pipeline
 priority: 97
 transformations:
     # Route to lowercase-normalized .caseless subfields for case-insensitive matching.
+    # file.path.caseless exists on Defend only (Sysmon file events lack it);
     # registry.path / dll.path / file.name have no .caseless on any source.
     - id: case_insensitive_string_fields
       type: field_name_mapping

salt/soc/files/soc/sigma_so_pipeline.yaml

@@ -26,16 +26,6 @@ transformations:
       type: set_state
       key: keep
       val: "_id, _index, _source"
-    # Not every source maps .caseless; EQL/ES|QL already match case-insensitively.
-    - id: caseless_to_parent_fields
-      type: field_name_mapping
-      mapping:
-        process.executable.caseless: process.executable
-        process.name.caseless: process.name
-        process.parent.executable.caseless: process.parent.executable
-        process.parent.name.caseless: process.parent.name
-        target.process.executable.caseless: target.process.executable
-        target.process.name.caseless: target.process.name
     - id: baseline_field_name_mapping
       type: field_name_mapping
       mapping:

salt/soc/soc_soc.yaml

@@ -862,15 +862,14 @@ soc:
             global: True
             forcedType: bool
           automations:
-            template:
-              description: Scheduled automations for the Onion AI assistant, managed from the Agent Studio. Each automation is stored under its own generated ID so its history and rollback are independent of every other automation.
-              global: True
-              advanced: False
-              readonlyUi: True
-              duplicates: True
-              forcedType: string
-              syntax: json
-              helpLink: onion-ai
+            description: Scheduled automations for the Onion AI assistant, managed from the Agent Studio.
+            global: True
+            advanced: True
+            readonlyUi: True
+            storage: db
+            forcedType: string
+            syntax: json
+            helpLink: onion-ai
           agents:
             description: Agent definitions for the Onion AI assistant, managed from the Agent Studio. An entry naming a system agent overrides only the fields an admin may change; everything else comes from the built-in definition.
             global: True