set module

salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json

@@ -29,7 +29,7 @@
               "\\.gz$"
             ],
             "include_files": [],
-            "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- drop_fields:\n    fields: [\"host\"]\n    ignore_missing: true\n- add_fields:\n    target: data_stream\n    fields:\n      type: logs\n      dataset: system.security\n- add_fields:\n    target: event\n    fields:\n      dataset: system.security\n      module: system\n      imported: true\n- add_fields:\n    target: \"@metadata\"\n    fields:\n        pipeline: logs-system.security-2.22.3\n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.sysmon_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.sysmon_operational\n          module: windows\n          imported: true\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.sysmon_operational-3.9.0\n- if:\n    equals:\n      winlog.channel: 'Application'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.application-2.22.3\n- if:\n    equals:\n      winlog.channel: 'System'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.system-2.22.3\n    \n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.powershell_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell_operational\n          module: windows\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.powershell_operational-3.9.0\n- add_fields:\n    target: data_stream\n    fields:\n        dataset: import",
+            "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- add_fields:\n    target: event\n    fields:\n      dataset: windows.forwarded\n      module: windows\n      imported: true\n- add_fields:\n    target: \"@metadata\"\n    fields:\n        pipeline: import.evtx\n- if:\n    equals:\n      winlog.channel: 'Security'\n  then: \n    - add_fields:\n        target: event\n        fields:\n          dataset: system.security\n          module: system\n- if:\n    equals:\n      winlog.channel: 'Windows PowerShell'\n  then: \n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell\n          module: windows\n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n  then: \n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.sysmon_operational\n          module: windows\n          imported: true\n- if:\n    equals:\n      winlog.channel: 'Application'\n  then: \n    - add_fields:\n        target: event\n        fields:\n          dataset: system.application\n          module: system\n- if:\n    equals:\n      winlog.channel: 'System'\n  then: \n    - add_fields:\n        target: event\n        fields:\n          dataset: system.system\n          module: system\n    \n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n  then: \n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell_operational\n          module: windows\n- add_fields:\n    target: data_stream\n    fields:\n        type: logs\n        dataset: import",
             "tags": [
               "import"
             ],

salt/elasticfleet/integration-defaults.map.jinja

@@ -30,14 +30,17 @@
     'azure_metrics.monitor': 'azure.monitor',
     'azure_metrics.storage_account': 'azure.storage_account',
     'azure_openai.metrics': 'azure.open_ai',
+    'beat.state': 'beats.stack_monitoring.state',
+    'beat.stats': 'beats.stack_monitoring.stats',
+    'enterprisesearch.health': 'enterprisesearch.stack_monitoring.health',
+    'enterprisesearch.stats': 'enterprisesearch.stack_monitoring.stats',
     'kibana.cluster_actions': 'kibana.stack_monitoring.cluster_actions',
     'kibana.cluster_rules': 'kibana.stack_monitoring.cluster_rules',
     'kibana.node_actions': 'kibana.stack_monitoring.node_actions',
     'kibana.node_rules': 'kibana.stack_monitoring.node_rules',
     'kibana.stats': 'kibana.stack_monitoring.stats',
     'kibana.status': 'kibana.stack_monitoring.status',
-    'logstash.node': 'logstash.stack_monitoring.node',
-    'logstash.node_cel': 'logstash.node',
+    'logstash.node_cel': 'logstash.stack_monitoring.node',
     'logstash.node_stats': 'logstash.stack_monitoring.node_stats',
     'synthetics.browser': 'synthetics-browser',
     'synthetics.browser_network': 'synthetics-browser.network',

salt/elasticsearch/files/ingest/global@custom

@@ -99,7 +99,7 @@
         },
         {
             "set": {
-                "if": "ctx.tags != null && ctx.tags.contains('import')",
+                "if": "ctx.tags != null && ctx.tags.contains('import') && ctx._index != null && ctx._index.startsWith('logs-import-')",
                 "override": true,
                 "field": "data_stream.dataset",
                 "value": "import"
@@ -107,7 +107,7 @@
         },
         {
             "set": {
-                "if": "ctx.tags != null && ctx.tags.contains('import')",
+                "if": "ctx.tags != null && ctx.tags.contains('import') && ctx._index != null && ctx._index.startsWith('logs-import-')",
                 "override": true,
                 "field": "data_stream.namespace",
                 "value": "so"

salt/elasticsearch/files/ingest/import.evtx

@@ -0,0 +1,31 @@
+{
+  "description" : "import.evtx: normalize imported EVTX and reroute to logs-<dataset>-import",
+  "processors" : [
+    { "script":  {
+        "description": "Host from the event, not the importing node",
+        "lang": "painless",
+        "source": "Map host = ['os': ['type': 'windows', 'family': 'windows', 'platform': 'windows']]; def cn = ctx.winlog?.computer_name; if (cn != null && cn.toString().length() > 0) { String name = cn.toString(); int dot = name.indexOf('.'); if (dot > 0) { name = name.substring(0, dot); } host.put('hostname', name); host.put('name', name.toLowerCase()); } ctx.host = host;"
+    } },
+    { "script":  {
+        "description": "String event IDs, as Winlogbeat sends",
+        "lang": "painless",
+        "source": "if (ctx.winlog?.event_id != null) { ctx.winlog.event_id = ctx.winlog.event_id.toString(); } if (ctx.event?.code != null) { ctx.event.code = ctx.event.code.toString(); }"
+    } },
+    { "script":  {
+        "description": "Unnamed <Data> to param1..N, as Winlogbeat",
+        "lang": "painless",
+        "if": "ctx.winlog?.event_data?.Data instanceof Map && ctx.winlog.event_data.Data['#text'] != null",
+        "source": "def t = ctx.winlog.event_data.Data['#text']; List vals = t instanceof List ? t : [t]; for (int i = 0; i < vals.size(); i++) { ctx.winlog.event_data['param' + (i + 1)] = vals.get(i); } ctx.winlog.event_data.remove('Data');"
+    } },
+    { "script":  {
+        "description": "String values and LF line endings, as Winlogbeat",
+        "lang": "painless",
+        "if": "ctx.winlog?.event_data instanceof Map || ctx.winlog?.user_data instanceof Map",
+        "source": "String lf = String.valueOf((char) 10); String crlf = String.valueOf((char) 13) + lf; for (def key : ['event_data', 'user_data']) { def m = ctx.winlog[key]; if (!(m instanceof Map)) { continue; } for (def e : m.entrySet()) { def v = e.getValue(); if (v instanceof String) { e.setValue(v.replace(crlf, lf)); } else if (v instanceof Number || v instanceof Boolean) { e.setValue(v.toString()); } } }"
+    } },
+    { "set":     { "description": "event.kind, as Winlogbeat", "field": "event.kind", "value": "event", "override": false } },
+    { "set":     { "field": "data_stream.dataset", "copy_from": "event.dataset", "override": true, "ignore_empty_value": true } },
+    { "set":     { "field": "data_stream.namespace", "value": "import", "override": true } },
+    { "reroute": { "dataset": "{{data_stream.dataset}}", "namespace": "{{data_stream.namespace}}" } }
+  ]
+}

salt/manager/tools/sbin/so-yaml.py

@@ -42,8 +42,7 @@ def loadYaml(filename):
     try:
         with open(filename, "r") as file:
             content = file.read()
-            loaded = yaml.safe_load(content)
-            return loaded if loaded is not None else {}
+            return yaml.safe_load(content)
     except FileNotFoundError:
         print(f"File not found: {filename}", file=sys.stderr)
         sys.exit(1)

salt/manager/tools/sbin/so-yaml_test.py

@@ -95,20 +95,6 @@ class TestRemove(unittest.TestCase):
         expected = "key1:\n  child1: 123\n  child2:\n    deep2: ab\nkey2: false\n"
         self.assertEqual(actual, expected)
 
-    def test_remove_empty_file(self):
-        filename = "/tmp/so-yaml_test-remove-empty.yaml"
-        file = open(filename, "w")
-        file.close()
-
-        code = soyaml.remove([filename, "key1"])
-        self.assertEqual(code, 0)
-
-        file = open(filename, "r")
-        actual = file.read()
-        file.close()
-
-        self.assertEqual(actual, "{}\n")
-
     def test_remove_missing_args(self):
         with patch('sys.exit', new=MagicMock()) as sysmock:
             with patch('sys.stderr', new=StringIO()) as mock_stderr:
@@ -308,36 +294,6 @@ class TestRemove(unittest.TestCase):
         expected = "key1:\n  child1: 123\n  child2:\n    deep1: 45\n    deep2: d\nkey2: false\nkey3:\n- e\n- f\n- g\n"
         self.assertEqual(actual, expected)
 
-    def test_add_empty_file(self):
-        filename = "/tmp/so-yaml_test-add-empty.yaml"
-        file = open(filename, "w")
-        file.close()
-
-        code = soyaml.add([filename, "telegraf.output", "BOTH"])
-        self.assertEqual(code, 0)
-
-        file = open(filename, "r")
-        actual = file.read()
-        file.close()
-
-        expected = "telegraf:\n  output: BOTH\n"
-        self.assertEqual(actual, expected)
-
-    def test_add_empty_file_simple(self):
-        filename = "/tmp/so-yaml_test-add-empty-simple.yaml"
-        file = open(filename, "w")
-        file.close()
-
-        code = soyaml.add([filename, "telegraf", "BOTH"])
-        self.assertEqual(code, 0)
-
-        file = open(filename, "r")
-        actual = file.read()
-        file.close()
-
-        expected = "telegraf: BOTH\n"
-        self.assertEqual(actual, expected)
-
     def test_replace_missing_arg(self):
         with patch('sys.exit', new=MagicMock()) as sysmock:
             with patch('sys.stderr', new=StringIO()) as mock_stderr:
@@ -390,21 +346,6 @@ class TestRemove(unittest.TestCase):
         expected = "key1:\n  child1: 123\n  child2:\n    deep1: 46\nkey2: false\nkey3:\n- e\n- f\n- g\n"
         self.assertEqual(actual, expected)
 
-    def test_replace_empty_file(self):
-        filename = "/tmp/so-yaml_test-replace-empty.yaml"
-        file = open(filename, "w")
-        file.close()
-
-        code = soyaml.replace([filename, "telegraf.output", "BOTH"])
-        self.assertEqual(code, 0)
-
-        file = open(filename, "r")
-        actual = file.read()
-        file.close()
-
-        expected = "telegraf:\n  output: BOTH\n"
-        self.assertEqual(actual, expected)
-
     def test_convert(self):
         self.assertEqual(soyaml.convertType("foo"), "foo")
         self.assertEqual(soyaml.convertType("foo.bar"), "foo.bar")
@@ -565,18 +506,6 @@ class TestRemove(unittest.TestCase):
             self.assertEqual(result, 2)
             self.assertEqual("", mock_stdout.getvalue())
 
-    def test_get_empty_file(self):
-        with patch('sys.stdout', new=StringIO()) as mock_stdout:
-            with patch('sys.stderr', new=StringIO()) as mock_stderr:
-                filename = "/tmp/so-yaml_test-get-empty.yaml"
-                file = open(filename, "w")
-                file.close()
-
-                result = soyaml.get([filename, "telegraf.output"])
-                self.assertEqual(result, 2)
-                self.assertEqual("", mock_stdout.getvalue())
-                self.assertIn("Key 'telegraf.output' not found by so-yaml.py", mock_stderr.getvalue())
-
     def test_get_usage(self):
         with patch('sys.exit', new=MagicMock()) as sysmock:
             with patch('sys.stderr', new=StringIO()) as mock_stderr:
@@ -1062,29 +991,3 @@ class TestLoadYaml(unittest.TestCase):
                     soyaml.loadYaml("/tmp/so-yaml_test-unreadable.yaml")
                     sysmock.assert_called_with(1)
                     self.assertIn("Error reading file", mock_stderr.getvalue())
-
-    def test_load_yaml_empty_file(self):
-        filename = "/tmp/so-yaml_test-load-empty.yaml"
-        file = open(filename, "w")
-        file.close()
-
-        result = soyaml.loadYaml(filename)
-        self.assertEqual(result, {})
-
-    def test_load_yaml_whitespace_only(self):
-        filename = "/tmp/so-yaml_test-load-whitespace.yaml"
-        file = open(filename, "w")
-        file.write("   \n\n  \n")
-        file.close()
-
-        result = soyaml.loadYaml(filename)
-        self.assertEqual(result, {})
-
-    def test_load_yaml_comments_only(self):
-        filename = "/tmp/so-yaml_test-load-comments.yaml"
-        file = open(filename, "w")
-        file.write("# Just a comment\n# Another comment\n")
-        file.close()
-
-        result = soyaml.loadYaml(filename)
-        self.assertEqual(result, {})

salt/manager/tools/sbin/soup

@@ -1183,13 +1183,6 @@ up_to_3.4.0() {
   echo "Removing so-kratos, so-hydra and so-soc so they are recreated on the soauth network."
   docker rm -f so-kratos so-hydra so-soc >> $SOUP_LOG 2>&1
 
-  for template in so-metrics-logstash.node so-metrics-logstash.stack_monitoring.node; do
-    if ! remove_elasticsearch_index_template "$template" "logstash node and node_cel index patterns reversed"; then
-      FINAL_MESSAGE_QUEUE+=("WARNING: Unable to automatically remove the $template index template. Addon integration templates may fail to load until it is removed:")
-      FINAL_MESSAGE_QUEUE+=("    - sudo so-elasticsearch-query _index_template/$template -XDELETE && so-checkin")
-    fi
-  done
-
   INSTALLEDVERSION=3.4.0
 }
 
@@ -1255,10 +1248,6 @@ valid_soauth_range() {
 }
 
 post_to_3.4.0() {
-  for idx in "metrics-logstash.node-default" "metrics-logstash.stack_monitoring.node-default"; do
-    rollover_index "$idx"
-  done
-
   set_postversion 3.4.0
 }
 ### 3.4.0 End ###

salt/soc/config.sls

@@ -120,30 +120,19 @@ crondetectionsbackup:
 
 socsigmafinalpipeline:
   file.managed:
-    - name: /opt/so/conf/soc/sigma_pipelines/sigma_final_pipeline.yml
+    - name: /opt/so/conf/soc/sigma_final_pipeline.yaml
     - source: salt://soc/files/soc/sigma_final_pipeline.yaml
     - user: 939
     - group: 939
     - mode: 600
-    - makedirs: True
 
-# sigma-cli loads every *.yml here; clean removes anything else
-socsigmapipelines:
-  file.recurse:
-    - name: /opt/so/conf/soc/sigma_pipelines
-    - source: salt://soc/files/soc/sigma_pipelines
+socsigmasopipeline:
+  file.managed:
+    - name: /opt/so/conf/soc/sigma_so_pipeline.yaml
+    - source: salt://soc/files/soc/sigma_so_pipeline.yaml
     - user: 939
     - group: 939
-    - file_mode: 600
-    - clean: True
-    - require:
-      - file: socsigmafinalpipeline
-
-socsigmapipelinesold:
-  file.absent:
-    - names:
-      - /opt/so/conf/soc/sigma_final_pipeline.yaml
-      - /opt/so/conf/soc/sigma_so_pipeline.yaml
+    - mode: 600
 
 socsigmaplaybookpipeline:
   file.managed:

salt/soc/enabled.sls

@@ -47,8 +47,9 @@ so-soc:
 {% endif %}
       - /opt/so/conf/soc/motd.md:/opt/sensoroni/html/motd.md:ro
       - /opt/so/conf/soc/banner.md:/opt/sensoroni/html/login/banner.md:ro
-      - /opt/so/conf/soc/sigma_pipelines:/opt/sensoroni/sigma_pipelines:ro
+      - /opt/so/conf/soc/sigma_so_pipeline.yaml:/opt/sensoroni/sigma_so_pipeline.yaml:ro
       - /opt/so/conf/soc/sigma_playbook_pipeline.yaml:/opt/sensoroni/sigma_playbook_pipeline.yaml:ro
+      - /opt/so/conf/soc/sigma_final_pipeline.yaml:/opt/sensoroni/sigma_final_pipeline.yaml:ro
       - /opt/so/conf/soc/playbook_placeholder_map.yaml:/opt/sensoroni/playbook_placeholder_map.yaml:ro
       - /opt/so/conf/soc/playbook_placeholder_map_custom.yaml:/opt/sensoroni/playbook_placeholder_map_custom.yaml:ro
       - /opt/so/conf/soc/custom.js:/opt/sensoroni/html/js/custom.js:ro
@@ -106,7 +107,6 @@ so-soc:
       - file: socclientsroles
       - file: socplaybookplaceholdermap
       - file: socplaybookplaceholdermapcustom
-      - file: socsigmapipelines
 
 delete_so-soc_so-status.disabled:
   file.uncomment:

salt/soc/files/soc/sigma_pipelines/sigma_esql_pipeline.yml

@@ -1,477 +0,0 @@
-name: Security Onion ES|QL Pipeline
-# ES|QL query settings
-priority: 92
-transformations:
-    - id: esql_default_index
-      type: set_state
-      key: index
-      val: .ds-logs-*
-    - id: esql_source_metadata
-      type: set_state
-      key: metadata
-      val: "_id, _index, _source"
-    - id: esql_source_keep
-      type: set_state
-      key: keep
-      val: "_id, _index, _source"
-    # unmapped fields read as null instead of failing the query
-    - id: esql_unmapped_fields
-      type: set_state
-      key: unmapped_fields
-      val: nullify
-    # FROM targets per logsource, any namespace; later entries win, correlations get the union
-    - id: esql_index_process_creation
-      type: set_state
-      key: index
-      val:
-        - .ds-logs-endpoint.events.process-*
-        - .ds-logs-windows.sysmon_operational-*
-        - .ds-logs-system.security-*
-        - .ds-logs-windows.powershell-*
-        - .ds-logs-windows.forwarded-*
-        - .ds-logs-sysmon_linux.log-*
-        - .ds-logs-auditd_manager.auditd-*
-        - .ds-logs-import-so-*
-      rule_conditions:
-      - type: logsource
-        category: process_creation
-    - id: esql_index_process_creation_windows
-      type: set_state
-      key: index
-      val:
-        - .ds-logs-endpoint.events.process-*
-        - .ds-logs-windows.sysmon_operational-*
-        - .ds-logs-system.security-*
-        - .ds-logs-windows.powershell-*
-        - .ds-logs-windows.forwarded-*
-        - .ds-logs-import-so-*
-      rule_conditions:
-      - type: logsource
-        product: windows
-        category: process_creation
-    - id: esql_index_process_creation_linux
-      type: set_state
-      key: index
-      val:
-        - .ds-logs-endpoint.events.process-*
-        - .ds-logs-sysmon_linux.log-*
-        - .ds-logs-auditd_manager.auditd-*
-        - .ds-logs-import-so-*
-      rule_conditions:
-      - type: logsource
-        product: linux
-        category: process_creation
-    - id: esql_index_process_creation_macos
-      type: set_state
-      key: index
-      val:
-        - .ds-logs-endpoint.events.process-*
-        - .ds-logs-import-so-*
-      rule_conditions:
-      - type: logsource
-        product: macos
-        category: process_creation
-    - id: esql_index_file
-      type: set_state
-      key: index
-      val:
-        - .ds-logs-endpoint.events.file-*
-        - .ds-logs-windows.sysmon_operational-*
-        - .ds-logs-windows.forwarded-*
-        - .ds-logs-sysmon_linux.log-*
-        - .ds-logs-import-so-*
-      rule_cond_op: or
-      rule_conditions:
-      - type: logsource
-        category: file_event
-      - type: logsource
-        category: file_delete
-      - type: logsource
-        category: file_rename
-      - type: logsource
-        category: file_change
-      - type: logsource
-        category: file_access
-    - id: esql_index_file_windows
-      type: set_state
-      key: index
-      val:
-        - .ds-logs-endpoint.events.file-*
-        - .ds-logs-windows.sysmon_operational-*
-        - .ds-logs-windows.forwarded-*
-        - .ds-logs-import-so-*
-      rule_cond_op: or
-      rule_conditions:
-      - type: logsource
-        product: windows
-        category: file_event
-      - type: logsource
-        product: windows
-        category: file_delete
-      - type: logsource
-        product: windows
-        category: file_rename
-      - type: logsource
-        product: windows
-        category: file_change
-      - type: logsource
-        product: windows
-        category: file_access
-    - id: esql_index_file_linux
-      type: set_state
-      key: index
-      val:
-        - .ds-logs-endpoint.events.file-*
-        - .ds-logs-sysmon_linux.log-*
-        - .ds-logs-import-so-*
-      rule_cond_op: or
-      rule_conditions:
-      - type: logsource
-        product: linux
-        category: file_event
-      - type: logsource
-        product: linux
-        category: file_delete
-      - type: logsource
-        product: linux
-        category: file_rename
-      - type: logsource
-        product: linux
-        category: file_change
-      - type: logsource
-        product: linux
-        category: file_access
-    - id: esql_index_file_macos
-      type: set_state
-      key: index
-      val:
-        - .ds-logs-endpoint.events.file-*
-        - .ds-logs-import-so-*
-      rule_cond_op: or
-      rule_conditions:
-      - type: logsource
-        product: macos
-        category: file_event
-      - type: logsource
-        product: macos
-        category: file_delete
-      - type: logsource
-        product: macos
-        category: file_rename
-      - type: logsource
-        product: macos
-        category: file_change
-      - type: logsource
-        product: macos
-        category: file_access
-    - id: esql_index_registry
-      type: set_state
-      key: index
-      val:
-        - .ds-logs-endpoint.events.registry-*
-        - .ds-logs-windows.sysmon_operational-*
-        - .ds-logs-windows.forwarded-*
-        - .ds-logs-import-so-*
-      rule_cond_op: or
-      rule_conditions:
-      - type: logsource
-        category: registry_set
-      - type: logsource
-        category: registry_add
-      - type: logsource
-        category: registry_delete
-      - type: logsource
-        category: registry_event
-    - id: esql_index_library
-      type: set_state
-      key: index
-      val:
-        - .ds-logs-endpoint.events.library-*
-        - .ds-logs-windows.sysmon_operational-*
-        - .ds-logs-windows.forwarded-*
-        - .ds-logs-import-so-*
-      rule_cond_op: or
-      rule_conditions:
-      - type: logsource
-        category: image_load
-      - type: logsource
-        category: driver_load
-    - id: esql_index_endpoint_network
-      type: set_state
-      key: index
-      val:
-        - .ds-logs-endpoint.events.network-*
-        - .ds-logs-windows.sysmon_operational-*
-        - .ds-logs-windows.forwarded-*
-        - .ds-logs-sysmon_linux.log-*
-        - .ds-logs-import-so-*
-      rule_cond_op: or
-      rule_conditions:
-      - type: logsource
-        category: network_connection
-      - type: logsource
-        category: dns_query
-    - id: esql_index_endpoint_network_windows
-      type: set_state
-      key: index
-      val:
-        - .ds-logs-endpoint.events.network-*
-        - .ds-logs-windows.sysmon_operational-*
-        - .ds-logs-windows.forwarded-*
-        - .ds-logs-import-so-*
-      rule_cond_op: or
-      rule_conditions:
-      - type: logsource
-        product: windows
-        category: network_connection
-      - type: logsource
-        product: windows
-        category: dns_query
-    - id: esql_index_endpoint_network_linux
-      type: set_state
-      key: index
-      val:
-        - .ds-logs-endpoint.events.network-*
-        - .ds-logs-sysmon_linux.log-*
-        - .ds-logs-import-so-*
-      rule_cond_op: or
-      rule_conditions:
-      - type: logsource
-        product: linux
-        category: network_connection
-      - type: logsource
-        product: linux
-        category: dns_query
-    - id: esql_index_endpoint_network_macos
-      type: set_state
-      key: index
-      val:
-        - .ds-logs-endpoint.events.network-*
-        - .ds-logs-import-so-*
-      rule_cond_op: or
-      rule_conditions:
-      - type: logsource
-        product: macos
-        category: network_connection
-      - type: logsource
-        product: macos
-        category: dns_query
-    - id: esql_index_sysmon_only
-      type: set_state
-      key: index
-      val:
-        - .ds-logs-windows.sysmon_operational-*
-        - .ds-logs-windows.forwarded-*
-        - .ds-logs-import-so-*
-      rule_cond_op: or
-      rule_conditions:
-      - type: logsource
-        category: process_access
-      - type: logsource
-        category: create_remote_thread
-      - type: logsource
-        category: pipe_created
-      - type: logsource
-        category: create_stream_hash
-      - type: logsource
-        category: wmi_event
-      - type: logsource
-        category: raw_access_thread
-      - type: logsource
-        category: process_tampering
-      - type: logsource
-        category: sysmon_status
-      - type: logsource
-        category: sysmon_error
-      - type: logsource
-        category: file_executable_detected
-      - type: logsource
-        category: file_block_executable
-      - type: logsource
-        category: file_block_shredding
-      - type: logsource
-        category: clipboard_capture
-      - type: logsource
-        product: windows
-        service: sysmon
-    - id: esql_index_ps_operational
-      type: set_state
-      key: index
-      val:
-        - .ds-logs-windows.powershell_operational-*
-        - .ds-logs-windows.forwarded-*
-        - .ds-logs-import-so-*
-      rule_cond_op: or
-      rule_conditions:
-      - type: logsource
-        category: ps_script
-      - type: logsource
-        category: ps_module
-      - type: logsource
-        product: windows
-        service: powershell
-    - id: esql_index_ps_classic
-      type: set_state
-      key: index
-      val:
-        - .ds-logs-windows.powershell-*
-        - .ds-logs-windows.forwarded-*
-        - .ds-logs-import-so-*
-      rule_cond_op: or
-      rule_conditions:
-      - type: logsource
-        category: ps_classic_start
-      - type: logsource
-        category: ps_classic_provider_start
-      - type: logsource
-        category: ps_classic_script
-      - type: logsource
-        product: windows
-        service: powershell-classic
-    - id: esql_index_win_security
-      type: set_state
-      key: index
-      val:
-        - .ds-logs-system.security-*
-        - .ds-logs-windows.forwarded-*
-        - .ds-logs-import-so-*
-      rule_conditions:
-      - type: logsource
-        product: windows
-        service: security
-    - id: esql_index_win_system
-      type: set_state
-      key: index
-      val:
-        - .ds-logs-system.system-*
-        - .ds-logs-windows.forwarded-*
-        - .ds-logs-import-so-*
-      rule_conditions:
-      - type: logsource
-        product: windows
-        service: system
-    - id: esql_index_win_application
-      type: set_state
-      key: index
-      val:
-        - .ds-logs-system.application-*
-        - .ds-logs-windows.forwarded-*
-        - .ds-logs-import-so-*
-      rule_conditions:
-      - type: logsource
-        product: windows
-        service: application
-    - id: esql_index_linux_auth
-      type: set_state
-      key: index
-      val:
-        - .ds-logs-system.auth-*
-        - .ds-logs-import-so-*
-      rule_cond_op: or
-      rule_conditions:
-      - type: logsource
-        product: linux
-        service: auth
-      - type: logsource
-        product: linux
-        service: sshd
-      - type: logsource
-        product: linux
-        service: sudo
-    - id: esql_index_linux_syslog
-      type: set_state
-      key: index
-      val:
-        - .ds-logs-system.syslog-*
-        - .ds-logs-syslog-so-*
-        - .ds-logs-import-so-*
-      rule_conditions:
-      - type: logsource
-        product: linux
-        service: syslog
-    - id: esql_index_linux_auditd
-      type: set_state
-      key: index
-      val:
-        - .ds-logs-auditd_manager.auditd-*
-        - .ds-logs-auditd.log-*
-        - .ds-logs-import-so-*
-      rule_conditions:
-      - type: logsource
-        product: linux
-        service: auditd
-    - id: esql_index_network
-      type: set_state
-      key: index
-      val:
-        - .ds-logs-zeek-so-*
-        - .ds-logs-suricata-so-*
-        - .ds-logs-suricata.alerts-so-*
-        - .ds-logs-endpoint.events.network-*
-        - .ds-logs-windows.sysmon_operational-*
-        - .ds-logs-sysmon_linux.log-*
-        - .ds-logs-windows.forwarded-*
-        - .ds-logs-import-so-*
-      rule_conditions:
-      - type: logsource
-        category: network
-    - id: esql_index_so_network
-      type: set_state
-      key: index
-      val:
-        - .ds-logs-zeek-so-*
-        - .ds-logs-suricata-so-*
-        - .ds-logs-import-so-*
-      rule_cond_op: or
-      rule_conditions:
-      - type: logsource
-        category: network
-        service: connection
-      - type: logsource
-        category: network
-        service: dns
-      - type: logsource
-        category: network
-        service: http
-      - type: logsource
-        category: network
-        service: file
-      - type: logsource
-        category: network
-        service: x509
-      - type: logsource
-        category: network
-        service: ssl
-      - type: logsource
-        category: network
-        service: ssh
-      - type: logsource
-        category: dns
-    - id: esql_index_zeek
-      type: set_state
-      key: index
-      val:
-        - .ds-logs-zeek-so-*
-        - .ds-logs-import-so-*
-      rule_conditions:
-      - type: logsource
-        product: zeek
-    - id: esql_index_opencanary
-      type: set_state
-      key: index
-      val:
-        - .ds-logs-idh-so-*
-        - .ds-logs-import-so-*
-      rule_conditions:
-      - type: logsource
-        product: opencanary
-    - id: esql_index_kratos
-      type: set_state
-      key: index
-      val:
-        - .ds-logs-kratos-so-*
-        - .ds-logs-import-so-*
-      rule_conditions:
-      - type: logsource
-        product: kratos
-

salt/soc/files/soc/sigma_so_pipeline.yaml

@@ -14,6 +14,18 @@ transformations:
         - process.args
         - related.ip
         - dns.resolved_ip
+    - id: esql_default_index
+      type: set_state
+      key: index
+      val: .ds-logs-*
+    - id: esql_source_metadata
+      type: set_state
+      key: metadata
+      val: "_id, _index, _source"
+    - id: esql_source_keep
+      type: set_state
+      key: keep
+      val: "_id, _index, _source"
     - id: baseline_field_name_mapping
       type: field_name_mapping
       mapping:

salt/soc/soc_soc.yaml

@@ -862,14 +862,15 @@ soc:
             global: True
             forcedType: bool
           automations:
-            description: Scheduled automations for the Onion AI assistant, managed from the Agent Studio.
-            global: True
-            advanced: True
-            readonlyUi: True
-            storage: db
-            forcedType: string
-            syntax: json
-            helpLink: onion-ai
+            template:
+              description: Scheduled automations for the Onion AI assistant, managed from the Agent Studio. Each automation is stored under its own generated ID so its history and rollback are independent of every other automation.
+              global: True
+              advanced: False
+              readonlyUi: True
+              duplicates: True
+              forcedType: string
+              syntax: json
+              helpLink: onion-ai
           agents:
             description: Agent definitions for the Onion AI assistant, managed from the Agent Studio. An entry naming a system agent overrides only the fields an admin may change; everything else comes from the built-in definition.
             global: True