filestream integration policy updates

pr/workflow changes

.github/DISCUSSION_TEMPLATE/3-0.yml

@@ -10,6 +10,7 @@ body:
       options:
         -
         - 3.0.0
+        - 3.1.0
         - Other (please provide detail below)
     validations:
       required: true

.github/pull_request_template.md

@@ -0,0 +1,22 @@
+## Description
+
+<!-- 
+Explain the purpose of the pull request. Be brief or detailed depending on the scope of the changes.
+-->
+
+## Related Issues
+
+<!-- 
+Optionally, list any related issues that this pull request addresses.
+-->
+
+## Checklist
+
+- [ ] I have read and followed the [CONTRIBUTING.md](https://github.com/Security-Onion-Solutions/securityonion/blob/3/main/CONTRIBUTING.md) file.
+- [ ] I have read and agree to the terms of the [Contributor License Agreement](https://securityonionsolutions.com/cla)
+
+## Questions or Comments
+
+<!--
+If you have any questions or comments about this pull request, add them here.
+-->

.github/workflows/contrib.yml

@@ -1,24 +0,0 @@
-name: contrib
-on:
-  issue_comment:
-    types: [created]
-  pull_request_target:
-    types: [opened,closed,synchronize]
-
-jobs:
-  CLAssistant:
-    runs-on: ubuntu-latest
-    steps:
-      - name: "Contributor Check"
-        if: (github.event.comment.body == 'recheck' || github.event.comment.body == 'I have read the CLA Document and I hereby sign the CLA') || github.event_name == 'pull_request_target'
-        uses: cla-assistant/github-action@v2.3.1
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          PERSONAL_ACCESS_TOKEN : ${{ secrets.PERSONAL_ACCESS_TOKEN }}
-        with:
-          path-to-signatures: 'signatures_v1.json'
-          path-to-document: 'https://securityonionsolutions.com/cla' 
-          allowlist: dependabot[bot],jertel,dougburks,TOoSmOotH,defensivedepth,m0duspwnens
-          remote-organization-name: Security-Onion-Solutions
-          remote-repository-name: licensing
-

CONTRIBUTING.md

@@ -23,7 +23,7 @@
 
 * Link the PR to the related issue, either using [keywords](https://docs.github.com/en/issues/tracking-your-work-with-issues/creating-issues/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword) in the PR description, or [manually](https://docs.github.com/en/issues/tracking-your-work-with-issues/creating-issues/linking-a-pull-request-to-an-issue#manually-linking-a-pull-request-to-an-issue).
 
-* **Pull requests should be opened against the `dev` branch of this repo**, and should clearly describe the problem and solution.
+* **Pull requests should be opened against the current `?/dev` branch of this repo**, and should clearly describe the problem and solution.
 
 * Be sure you have tested your changes and are confident they will not break other parts of the product.
 

DOWNLOAD_AND_VERIFY_ISO.md

@@ -1,46 +1,46 @@
-### 2.4.210-20260302 ISO image released on 2026/03/02
+### 3.0.0-20260331 ISO image released on 2026/03/31
 
 
 ### Download and Verify
 
-2.4.210-20260302 ISO image:  
-https://download.securityonion.net/file/securityonion/securityonion-2.4.210-20260302.iso
+3.0.0-20260331 ISO image:  
+https://download.securityonion.net/file/securityonion/securityonion-3.0.0-20260331.iso
  
-MD5: 575F316981891EBED2EE4E1F42A1F016  
-SHA1: 600945E8823221CBC5F1C056084A71355308227E  
-SHA256: A6AA6471125F07FA6E2796430E94BEAFDEF728E833E9728FDFA7106351EBC47E  
+MD5: ECD318A1662A6FDE0EF213F5A9BD4B07  
+SHA1: E55BE314440CCF3392DC0B06BC5E270B43176D9C  
+SHA256: 7FC47405E335CBE5C2B6C51FE7AC60248F35CBE504907B8B5A33822B23F8F4D5  
 
 Signature for ISO image:  
-https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.210-20260302.iso.sig
+https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.0.0-20260331.iso.sig
 
 Signing key:  
-https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.4/main/KEYS  
+https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/3/main/KEYS  
 
 For example, here are the steps you can use on most Linux distributions to download and verify our Security Onion ISO image.
 
 Download and import the signing key:  
 ```
-wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.4/main/KEYS -O - | gpg --import -  
+wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/3/main/KEYS -O - | gpg --import -  
 ```
 
 Download the signature file for the ISO:  
 ```
-wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.210-20260302.iso.sig
+wget https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.0.0-20260331.iso.sig
 ```
 
 Download the ISO image:  
 ```
-wget https://download.securityonion.net/file/securityonion/securityonion-2.4.210-20260302.iso
+wget https://download.securityonion.net/file/securityonion/securityonion-3.0.0-20260331.iso
 ```
 
 Verify the downloaded ISO image using the signature file:  
 ```
-gpg --verify securityonion-2.4.210-20260302.iso.sig securityonion-2.4.210-20260302.iso
+gpg --verify securityonion-3.0.0-20260331.iso.sig securityonion-3.0.0-20260331.iso
 ```
 
 The output should show "Good signature" and the Primary key fingerprint should match what's shown below:
 ```
-gpg: Signature made Mon 02 Mar 2026 11:55:24 AM EST using RSA key ID FE507013
+gpg: Signature made Mon 30 Mar 2026 06:22:14 PM EDT using RSA key ID FE507013
 gpg: Good signature from "Security Onion Solutions, LLC <info@securityonionsolutions.com>"
 gpg: WARNING: This key is not certified with a trusted signature!
 gpg:          There is no indication that the signature belongs to the owner.

VERSION

@@ -1 +1 @@
-3.0.0
+3.0.0-foxtrot

salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json

@@ -5,7 +5,7 @@
   "package": {
     "name": "endpoint",
     "title": "Elastic Defend",
-    "version": "9.0.2",
+    "version": "9.3.0",
     "requires_root": true
   },
   "enabled": true,

salt/elasticfleet/files/integrations/grid-nodes_general/elastic-agent-monitor.json

@@ -6,21 +6,23 @@
   "name": "agent-monitor",
   "namespace": "",
   "description": "",
+  "policy_id": "so-grid-nodes_general",
   "policy_ids": [
     "so-grid-nodes_general"
   ],
-  "output_id": null,
   "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.generic": {
+        "filestream.filestream": {
           "enabled": true,
           "vars": {
             "paths": [
               "/opt/so/log/agents/agent-monitor.log"
             ],
+            "compression_gzip": false,
+            "use_logs_stream": false,
             "data_stream.dataset": "agentmonitor",
             "pipeline": "elasticagent.monitor",
             "parsers": "",
@@ -34,15 +36,16 @@
             "ignore_older": "72h",
             "clean_inactive": -1,
             "harvester_limit": 0,
-            "fingerprint": true,
+            "fingerprint": false,
             "fingerprint_offset": 0,
-            "fingerprint_length": 64,
-            "file_identity_native": false,
+            "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": []
+            "include_lines": [],
+            "delete_enabled": false
           }
         }
       }
     }
-  }
+  },
+  "force": true
 }

salt/elasticfleet/files/integrations/grid-nodes_general/hydra-logs.json

@@ -4,19 +4,25 @@
     "version": ""
   },
   "name": "hydra-logs",
+  "namespace": "so",
   "description": "Hydra logs",
   "policy_id": "so-grid-nodes_general",
-  "namespace": "so",
+  "policy_ids": [
+    "so-grid-nodes_general"
+  ],
+  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.generic": {
+        "filestream.filestream": {
           "enabled": true,
           "vars": {
             "paths": [
               "/opt/so/log/hydra/hydra.log"
             ],
+            "compression_gzip": false,
+            "use_logs_stream": false,
             "data_stream.dataset": "hydra",
             "pipeline": "hydra",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -34,10 +40,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
-            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": []
+            "include_lines": [],
+            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/idh-logs.json

@@ -4,19 +4,25 @@
     "version": ""
   },
   "name": "idh-logs",
+  "namespace": "so",
   "description": "IDH integration",
   "policy_id": "so-grid-nodes_general",
-  "namespace": "so",
+  "policy_ids": [
+    "so-grid-nodes_general"
+  ],
+  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.generic": {
+        "filestream.filestream": {
           "enabled": true,
           "vars": {
             "paths": [
               "/nsm/idh/opencanary.log"
             ],
+            "compression_gzip": false,
+            "use_logs_stream": false,
             "data_stream.dataset": "idh",
             "pipeline": "common",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -31,10 +37,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
-            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": []
+            "include_lines": [],
+            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json

@@ -4,26 +4,32 @@
     "version": ""
   },
   "name": "import-evtx-logs",
+  "namespace": "so",
   "description": "Import Windows EVTX logs",
   "policy_id": "so-grid-nodes_general",
-  "namespace": "so",
+  "policy_ids": [
+    "so-grid-nodes_general"
+  ],
+  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.generic": {
+        "filestream.filestream": {
           "enabled": true,
           "vars": {
             "paths": [
               "/nsm/import/*/evtx/*.json"
             ],
+            "compression_gzip": false,
+            "use_logs_stream": false,
             "data_stream.dataset": "import",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
             "exclude_files": [
               "\\.gz$"
             ],
             "include_files": [],
-            "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- drop_fields:\n    fields: [\"host\"]\n    ignore_missing: true\n- add_fields:\n    target: data_stream\n    fields:\n      type: logs\n      dataset: system.security\n- add_fields:\n    target: event\n    fields:\n      dataset: system.security\n      module: system\n      imported: true\n- add_fields:\n    target: \"@metadata\"\n    fields:\n        pipeline: logs-system.security-2.6.1\n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.sysmon_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.sysmon_operational\n          module: windows\n          imported: true\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.sysmon_operational-3.1.2\n- if:\n    equals:\n      winlog.channel: 'Application'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.application-2.6.1\n- if:\n    equals:\n      winlog.channel: 'System'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.system-2.6.1\n    \n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.powershell_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell_operational\n          module: windows\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.powershell_operational-3.1.2\n- add_fields:\n    target: data_stream\n    fields:\n        dataset: import",
+            "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- drop_fields:\n    fields: [\"host\"]\n    ignore_missing: true\n- add_fields:\n    target: data_stream\n    fields:\n      type: logs\n      dataset: system.security\n- add_fields:\n    target: event\n    fields:\n      dataset: system.security\n      module: system\n      imported: true\n- add_fields:\n    target: \"@metadata\"\n    fields:\n        pipeline: logs-system.security-2.13.0\n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.sysmon_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.sysmon_operational\n          module: windows\n          imported: true\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.sysmon_operational-3.6.0\n- if:\n    equals:\n      winlog.channel: 'Application'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.application-2.13.0\n- if:\n    equals:\n      winlog.channel: 'System'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.system-2.13.0\n    \n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.powershell_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell_operational\n          module: windows\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.powershell_operational-3.6.0\n- add_fields:\n    target: data_stream\n    fields:\n        dataset: import",
             "tags": [
               "import"
             ],
@@ -33,10 +39,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
-            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": []
+            "include_lines": [],
+            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/import-suricata-logs.json

@@ -4,19 +4,25 @@
     "version": ""
   },
   "name": "import-suricata-logs",
+  "namespace": "so",
   "description": "Import Suricata logs",
   "policy_id": "so-grid-nodes_general",
-  "namespace": "so",
+  "policy_ids": [
+    "so-grid-nodes_general"
+  ],
+  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.generic": {
+        "filestream.filestream": {
           "enabled": true,
           "vars": {
             "paths": [
               "/nsm/import/*/suricata/eve*.json"
             ],
+            "compression_gzip": false,
+            "use_logs_stream": false,
             "data_stream.dataset": "import",
             "pipeline": "suricata.common",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -32,10 +38,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
-            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": []
+            "include_lines": [],
+            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/rita-logs.json

@@ -4,14 +4,18 @@
     "version": ""
   },
   "name": "rita-logs",
+  "namespace": "so",
   "description": "RITA Logs",
   "policy_id": "so-grid-nodes_general",
-  "namespace": "so",
+  "policy_ids": [
+    "so-grid-nodes_general"
+  ],
+  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.generic": {
+        "filestream.filestream": {
           "enabled": true,
           "vars": {
             "paths": [
@@ -19,6 +23,8 @@
               "/nsm/rita/exploded-dns.csv",
               "/nsm/rita/long-connections.csv"
             ],
+            "compression_gzip": false,
+            "use_logs_stream": false,
             "data_stream.dataset": "rita",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
             "exclude_files": [
@@ -33,10 +39,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
-            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": []
+            "include_lines": [],
+            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/so-ip-mappings.json

@@ -4,19 +4,25 @@
     "version": ""
   },
   "name": "so-ip-mappings",
+  "namespace": "so",
   "description": "IP Description mappings",
   "policy_id": "so-grid-nodes_general",
-  "namespace": "so",
+  "policy_ids": [
+    "so-grid-nodes_general"
+  ],
+  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.generic": {
+        "filestream.filestream": {
           "enabled": true,
           "vars": {
             "paths": [
               "/nsm/custom-mappings/ip-descriptions.csv"
             ],
+            "compression_gzip": false,
+            "use_logs_stream": false,
             "data_stream.dataset": "hostnamemappings",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
             "exclude_files": [
@@ -32,10 +38,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
-            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": []
+            "include_lines": [],
+            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/soc-auth-sync-logs.json

@@ -4,19 +4,25 @@
     "version": ""
   },
   "name": "soc-auth-sync-logs",
+  "namespace": "so",
   "description": "Security Onion - Elastic Auth Sync - Logs",
   "policy_id": "so-grid-nodes_general",
-  "namespace": "so",
+  "policy_ids": [
+    "so-grid-nodes_general"
+  ],
+  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.generic": {
+        "filestream.filestream": {
           "enabled": true,
           "vars": {
             "paths": [
               "/opt/so/log/soc/sync.log"
             ],
+            "compression_gzip": false,
+            "use_logs_stream": false,
             "data_stream.dataset": "soc",
             "pipeline": "common",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -31,10 +37,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
-            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": []
+            "include_lines": [],
+            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/soc-detections-logs.json

@@ -4,20 +4,26 @@
     "version": ""
   },
   "name": "soc-detections-logs",
+  "namespace": "so",
   "description": "Security Onion Console - Detections Logs",
   "policy_id": "so-grid-nodes_general",
-  "namespace": "so",
+  "policy_ids": [
+    "so-grid-nodes_general"
+  ],
+  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.generic": {
+        "filestream.filestream": {
           "enabled": true,
           "vars": {
             "paths": [
               "/opt/so/log/soc/detections_runtime-status_sigma.log",
               "/opt/so/log/soc/detections_runtime-status_yara.log"
             ],
+            "compression_gzip": false,
+            "use_logs_stream": false,
             "data_stream.dataset": "soc",
             "pipeline": "common",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -35,10 +41,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
-            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": []
+            "include_lines": [],
+            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/soc-salt-relay-logs.json

@@ -4,19 +4,25 @@
     "version": ""
   },
   "name": "soc-salt-relay-logs",
+  "namespace": "so",
   "description": "Security Onion - Salt Relay - Logs",
   "policy_id": "so-grid-nodes_general",
-  "namespace": "so",
+  "policy_ids": [
+    "so-grid-nodes_general"
+  ],
+  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.generic": {
+        "filestream.filestream": {
           "enabled": true,
           "vars": {
             "paths": [
               "/opt/so/log/soc/salt-relay.log"
             ],
+            "compression_gzip": false,
+            "use_logs_stream": false,
             "data_stream.dataset": "soc",
             "pipeline": "common",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -33,10 +39,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
-            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": []
+            "include_lines": [],
+            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/soc-sensoroni-logs.json

@@ -4,19 +4,25 @@
     "version": ""
   },
   "name": "soc-sensoroni-logs",
+  "namespace": "so",
   "description": "Security Onion - Sensoroni - Logs",
   "policy_id": "so-grid-nodes_general",
-  "namespace": "so",
+  "policy_ids": [
+    "so-grid-nodes_general"
+  ],
+  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.generic": {
+        "filestream.filestream": {
           "enabled": true,
           "vars": {
             "paths": [
               "/opt/so/log/sensoroni/sensoroni.log"
             ],
+            "compression_gzip": false,
+            "use_logs_stream": false,
             "data_stream.dataset": "soc",
             "pipeline": "common",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -31,10 +37,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
-            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": []
+            "include_lines": [],
+            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/soc-server-logs.json

@@ -4,19 +4,25 @@
     "version": ""
   },
   "name": "soc-server-logs",
+  "namespace": "so",
   "description": "Security Onion Console Logs",
   "policy_id": "so-grid-nodes_general",
-  "namespace": "so",
+  "policy_ids": [
+    "so-grid-nodes_general"
+  ],
+  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.generic": {
+        "filestream.filestream": {
           "enabled": true,
           "vars": {
             "paths": [
               "/opt/so/log/soc/sensoroni-server.log"
             ],
+            "compression_gzip": false,
+            "use_logs_stream": false,
             "data_stream.dataset": "soc",
             "pipeline": "common",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -33,10 +39,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
-            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": []
+            "include_lines": [],
+            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/strelka-logs.json

@@ -4,19 +4,25 @@
     "version": ""
   },
   "name": "strelka-logs",
+  "namespace": "so",
   "description": "Strelka Logs",
   "policy_id": "so-grid-nodes_general",
-  "namespace": "so",
+  "policy_ids": [
+    "so-grid-nodes_general"
+  ],
+  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.generic": {
+        "filestream.filestream": {
           "enabled": true,
           "vars": {
             "paths": [
               "/nsm/strelka/log/strelka.log"
             ],
+            "compression_gzip": false,
+            "use_logs_stream": false,
             "data_stream.dataset": "strelka",
             "pipeline": "strelka.file",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -31,10 +37,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
-            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": []
+            "include_lines": [],
+            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/suricata-logs.json

@@ -4,19 +4,25 @@
     "version": ""
   },
   "name": "suricata-logs",
+  "namespace": "so",
   "description": "Suricata integration",
   "policy_id": "so-grid-nodes_general",
-  "namespace": "so",
+  "policy_ids": [
+    "so-grid-nodes_general"
+  ],
+  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.generic": {
+        "filestream.filestream": {
           "enabled": true,
           "vars": {
             "paths": [
               "/nsm/suricata/eve*.json"
             ],
+            "compression_gzip": false,
+            "use_logs_stream": false,
             "data_stream.dataset": "suricata",
             "pipeline": "suricata.common",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -31,10 +37,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
-            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": []
+            "include_lines": [],
+            "delete_enabled": false
           }
         }
       }

salt/elasticsearch/defaults.yaml

@@ -1,6 +1,6 @@
 elasticsearch:
   enabled: false
-  version: 9.0.8
+  version: 9.3.2
   index_clean: true
   vm:
     max_map_count: 1048576

salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.1

@@ -10,24 +10,28 @@
   "processors": [
     {
       "set": {
+        "tag": "set_ecs_version_f5923549",
         "field": "ecs.version",
         "value": "8.17.0"
       }
     },
     {
       "set": {
+        "tag": "set_observer_vendor_ad9d35cc",
         "field": "observer.vendor",
         "value": "netgate"
       }
     },
     {
       "set": {
+        "tag": "set_observer_type_5dddf3ba",
         "field": "observer.type",
         "value": "firewall"
       }
     },
     {
       "rename": {
+        "tag": "rename_message_to_event_original_56a77271",
         "field": "message",
         "target_field": "event.original",
         "ignore_missing": true,
@@ -36,12 +40,14 @@
     },
     {
       "set": {
+        "tag": "set_event_kind_de80643c",
         "field": "event.kind",
         "value": "event"
       }
     },
     {
       "set": {
+        "tag": "set_event_timezone_4ca44cac",
         "field": "event.timezone",
         "value": "{{{_tmp.tz_offset}}}",
         "if": "ctx._tmp?.tz_offset != null && ctx._tmp?.tz_offset != 'local'"
@@ -49,6 +55,7 @@
     },
     {
       "grok": {
+        "tag": "grok_event_original_27d9c8c7",
         "description": "Parse syslog header",
         "field": "event.original",
         "patterns": [
@@ -72,6 +79,7 @@
     },
     {
       "date": {
+        "tag": "date__tmp_timestamp8601_to_timestamp_6ac9d3ce",
         "if": "ctx._tmp.timestamp8601 != null",
         "field": "_tmp.timestamp8601",
         "target_field": "@timestamp",
@@ -82,6 +90,7 @@
     },
     {
       "date": {
+        "tag": "date__tmp_timestamp_to_timestamp_f21e536e",
         "if": "ctx.event?.timezone != null && ctx._tmp?.timestamp != null",
         "field": "_tmp.timestamp",
         "target_field": "@timestamp",
@@ -95,6 +104,7 @@
     },
     {
       "grok": {
+        "tag": "grok_process_name_cef3d489",
         "description": "Set Event Provider",
         "field": "process.name",
         "patterns": [
@@ -107,71 +117,83 @@
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.23.1-firewall",
+        "tag": "pipeline_e16851a7",
+        "name": "logs-pfsense.log-1.25.1-firewall",
         "if": "ctx.event.provider == 'filterlog'"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.23.1-openvpn",
+        "tag": "pipeline_828590b5",
+        "name": "logs-pfsense.log-1.25.1-openvpn",
         "if": "ctx.event.provider == 'openvpn'"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.23.1-ipsec",
+        "tag": "pipeline_9d37039c",
+        "name": "logs-pfsense.log-1.25.1-ipsec",
         "if": "ctx.event.provider == 'charon'"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.23.1-dhcp",
+        "tag": "pipeline_ad56bbca",
+        "name": "logs-pfsense.log-1.25.1-dhcp",
         "if": "[\"dhcpd\", \"dhclient\", \"dhcp6c\"].contains(ctx.event.provider)"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.23.1-unbound",
+        "tag": "pipeline_dd85553d",
+        "name": "logs-pfsense.log-1.25.1-unbound",
         "if": "ctx.event.provider == 'unbound'"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.23.1-haproxy",
+        "tag": "pipeline_720ed255",
+        "name": "logs-pfsense.log-1.25.1-haproxy",
         "if": "ctx.event.provider == 'haproxy'"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.23.1-php-fpm",
+        "tag": "pipeline_456beba5",
+        "name": "logs-pfsense.log-1.25.1-php-fpm",
         "if": "ctx.event.provider == 'php-fpm'"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.23.1-squid",
+        "tag": "pipeline_a0d89375",
+        "name": "logs-pfsense.log-1.25.1-squid",
         "if": "ctx.event.provider == 'squid'"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.23.1-snort",
+        "tag": "pipeline_c2f1ed55",
+        "name": "logs-pfsense.log-1.25.1-snort",
         "if": "ctx.event.provider == 'snort'"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.23.1-suricata",
+        "tag":"pipeline_33db1c9e",
+        "name": "logs-pfsense.log-1.25.1-suricata",
         "if": "ctx.event.provider == 'suricata'"
       }
     },
     {
       "drop": {
+        "tag": "drop_9d7c46f8",
         "if": "![\"filterlog\", \"openvpn\", \"charon\", \"dhcpd\", \"dhclient\", \"dhcp6c\", \"unbound\", \"haproxy\", \"php-fpm\", \"squid\", \"snort\", \"suricata\"].contains(ctx.event?.provider)"
       }
     },
     {
       "append": {
+        "tag": "append_event_category_4780a983",
         "field": "event.category",
         "value": "network",
         "if": "ctx.network != null"
@@ -179,6 +201,7 @@
     },
     {
       "convert": {
+        "tag": "convert_source_address_to_source_ip_f5632a20",
         "field": "source.address",
         "target_field": "source.ip",
         "type": "ip",
@@ -188,6 +211,7 @@
     },
     {
       "convert": {
+        "tag": "convert_destination_address_to_destination_ip_f1388f0c",
         "field": "destination.address",
         "target_field": "destination.ip",
         "type": "ip",
@@ -197,6 +221,7 @@
     },
     {
       "set": {
+        "tag": "set_network_type_1f1d940a",
         "field": "network.type",
         "value": "ipv6",
         "if": "ctx.source?.ip != null && ctx.source.ip.contains(\":\")"
@@ -204,6 +229,7 @@
     },
     {
       "set": {
+        "tag": "set_network_type_69deca38",
         "field": "network.type",
         "value": "ipv4",
         "if": "ctx.source?.ip != null && ctx.source.ip.contains(\".\")"
@@ -211,6 +237,7 @@
     },
     {
       "geoip": {
+        "tag": "geoip_source_ip_to_source_geo_da2e41b2",
         "field": "source.ip",
         "target_field": "source.geo",
         "ignore_missing": true
@@ -218,6 +245,7 @@
     },
     {
       "geoip": {
+        "tag": "geoip_destination_ip_to_destination_geo_ab5e2968",
         "field": "destination.ip",
         "target_field": "destination.geo",
         "ignore_missing": true
@@ -225,6 +253,7 @@
     },
     {
       "geoip": {
+        "tag": "geoip_source_ip_to_source_as_28d69883",
         "ignore_missing": true,
         "database_file": "GeoLite2-ASN.mmdb",
         "field": "source.ip",
@@ -237,6 +266,7 @@
     },
     {
       "geoip": {
+        "tag": "geoip_destination_ip_to_destination_as_8a007787",
         "database_file": "GeoLite2-ASN.mmdb",
         "field": "destination.ip",
         "target_field": "destination.as",
@@ -249,6 +279,7 @@
     },
     {
       "rename": {
+        "tag": "rename_source_as_asn_to_source_as_number_a917047d",
         "field": "source.as.asn",
         "target_field": "source.as.number",
         "ignore_missing": true
@@ -256,6 +287,7 @@
     },
     {
       "rename": {
+        "tag": "rename_source_as_organization_name_to_source_as_organization_name_f1362d0b",
         "field": "source.as.organization_name",
         "target_field": "source.as.organization.name",
         "ignore_missing": true
@@ -263,6 +295,7 @@
     },
     {
       "rename": {
+        "tag": "rename_destination_as_asn_to_destination_as_number_3b459fcd",
         "field": "destination.as.asn",
         "target_field": "destination.as.number",
         "ignore_missing": true
@@ -270,6 +303,7 @@
     },
     {
       "rename": {
+        "tag": "rename_destination_as_organization_name_to_destination_as_organization_name_814bd459",
         "field": "destination.as.organization_name",
         "target_field": "destination.as.organization.name",
         "ignore_missing": true
@@ -277,12 +311,14 @@
     },
     {
       "community_id": {
+        "tag": "community_id_d2308e7a",
         "target_field": "network.community_id",
         "ignore_failure": true
       }
     },
     {
       "grok": {
+        "tag": "grok_observer_ingress_interface_name_968018d3",
         "field": "observer.ingress.interface.name",
         "patterns": [
           "%{DATA}.%{NONNEGINT:observer.ingress.vlan.id}"
@@ -293,6 +329,7 @@
     },
     {
       "set": {
+        "tag": "set_network_vlan_id_efd4d96a",
         "field": "network.vlan.id",
         "copy_from": "observer.ingress.vlan.id",
         "ignore_empty_value": true
@@ -300,6 +337,7 @@
     },
     {
       "append": {
+        "tag": "append_related_ip_c1a6356b",
         "field": "related.ip",
         "value": "{{{destination.ip}}}",
         "allow_duplicates": false,
@@ -308,6 +346,7 @@
     },
     {
       "append": {
+        "tag": "append_related_ip_8121c591",
         "field": "related.ip",
         "value": "{{{source.ip}}}",
         "allow_duplicates": false,
@@ -316,6 +355,7 @@
     },
     {
       "append": {
+        "tag": "append_related_ip_53b62ed8",
         "field": "related.ip",
         "value": "{{{source.nat.ip}}}",
         "allow_duplicates": false,
@@ -324,6 +364,7 @@
     },
     {
       "append": {
+        "tag": "append_related_hosts_6f162628",
         "field": "related.hosts",
         "value": "{{{destination.domain}}}",
         "if": "ctx.destination?.domain != null"
@@ -331,6 +372,7 @@
     },
     {
       "append": {
+        "tag": "append_related_user_c036eec2",
         "field": "related.user",
         "value": "{{{user.name}}}",
         "if": "ctx.user?.name != null"
@@ -338,6 +380,7 @@
     },
     {
       "set": {
+        "tag": "set_network_direction_cb1e3125",
         "field": "network.direction",
         "value": "{{{network.direction}}}bound",
         "if": "ctx.network?.direction != null && ctx.network?.direction =~ /^(in|out)$/"
@@ -345,6 +388,7 @@
     },
     {
       "remove": {
+        "tag": "remove_a82e20f2",
         "field": [
           "_tmp"
         ],
@@ -353,11 +397,21 @@
     },
     {
       "script": {
+        "tag": "script_a7f2c062",
         "lang": "painless",
         "description": "This script processor iterates over the whole document to remove fields with null values.",
         "source": "void handleMap(Map map) {\n  for (def x : map.values()) {\n    if (x instanceof Map) {\n        handleMap(x);\n    } else if (x instanceof List) {\n        handleList(x);\n    }\n  }\n  map.values().removeIf(v -> v == null || (v instanceof String && v == \"-\"));\n}\nvoid handleList(List list) {\n  for (def x : list) {\n      if (x instanceof Map) {\n          handleMap(x);\n      } else if (x instanceof List) {\n          handleList(x);\n      }\n  }\n}\nhandleMap(ctx);\n"
       }
     },
+    {
+      "append": {
+        "tag": "append_preserve_original_event_on_error",
+        "field": "tags",
+        "value": "preserve_original_event",
+        "allow_duplicates": false,
+        "if": "ctx.error?.message != null"
+      }
+    },
     {
       "pipeline": {
         "name": "global@custom",
@@ -405,7 +459,14 @@
     {
       "append": {
         "field": "error.message",
-        "value": "{{{ _ingest.on_failure_message }}}"
+        "value": "Processor '{{{ _ingest.on_failure_processor_type }}}' {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}'"
+      }
+    },
+    {
+      "append": {
+        "field": "tags",
+        "value": "preserve_original_event",
+        "allow_duplicates": false
       }
     }
   ]

salt/kibana/tools/sbin_jinja/so-kibana-space-defaults

@@ -9,5 +9,5 @@ SESSIONCOOKIE=$(curl -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http:
 # Disable certain Features from showing up in the Kibana UI
 echo
 echo "Setting up default Kibana Space:"
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["ml","enterpriseSearch","logs","infrastructure","apm","uptime","monitoring","stackAlerts","actions","securitySolutionCasesV3","inventory","dataQuality","searchSynonyms","enterpriseSearchApplications","enterpriseSearchAnalytics","securitySolutionTimeline","securitySolutionNotes","entityManager"]} ' >> /opt/so/log/kibana/misc.log
+curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["ml","enterpriseSearch","logs","infrastructure","apm","uptime","monitoring","stackAlerts","actions","securitySolutionCasesV3","inventory","dataQuality","searchSynonyms","searchQueryRules","enterpriseSearchApplications","enterpriseSearchAnalytics","securitySolutionTimeline","securitySolutionNotes","securitySolutionRulesV1","entityManager","streams","cloudConnect","slo"]} ' >> /opt/so/log/kibana/misc.log
 echo

salt/manager/tools/sbin/soup

@@ -305,7 +305,7 @@ clone_to_tmp() {
   # Make a temp location for the files
   mkdir -p /tmp/sogh
   cd /tmp/sogh
-  SOUP_BRANCH="-b 2.4/main"
+  SOUP_BRANCH="-b 3/main"
   if [ -n "$BRANCH" ]; then
     SOUP_BRANCH="-b $BRANCH"
   fi

salt/suricata/tools/sbin_jinja/so-suricata-testrule

@@ -27,7 +27,7 @@ echo ""
 sleep 3
 
 rm -rf /tmp/nids-testing/output
-mkdir -p /tmp/nids-testing/output
+mkdir -p /tmp/nids-testing/output/suripcap
 chown suricata:socore /tmp/nids-testing/output
 mkdir -p /tmp/nids-testing/rules
 
@@ -45,7 +45,7 @@ echo "==== Begin Suricata Output ==="
     -v /opt/so/conf/suricata/bpf:/etc/suricata/bpf:ro \
     -v /tmp/nids-testing/output/:/nsm/:rw \
     {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-suricata:{{ VERSION }} \
-    --runmode single -v -k none -r /input.pcap -l /tmp --init-errors-fatal --set outputs.6.pcap-log.enabled=no
+    --runmode single -v -k none -r /input.pcap -l /tmp --init-errors-fatal
 echo "==== End Suricata Output ==="
 
 echo ""